From c70fbc671ef6bfdb64a174e12f566e0130f02302 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:40:30 +0200 Subject: [PATCH] feat: hls content encoding support and shared session between different targets (#789) * feat: key shared hls sessions by origin id * feat: support hls content encoding * fix: coderabbitai findings * Fix: Custom headers are lost on provider:// In the manual redirect path, the abstract provider:// URL is compared against the resolved HTTP URL. This causes the request to be marked as "scrubbed" before the first real redirect even occurs, stripping out headers like X-API-Key. --- backend/Cargo.toml | 2 +- backend/src/api/api_utils.rs | 1103 ++++++----- .../api/endpoints/custom_video_stream_api.rs | 13 +- backend/src/api/endpoints/hls_api.rs | 1612 ++++++++++------ backend/src/api/endpoints/m3u_api.rs | 13 +- backend/src/api/endpoints/xtream_api.rs | 21 +- backend/src/api/model/hls_cache/cache.rs | 134 +- backend/src/api/model/hls_cache/gc.rs | 11 +- backend/src/api/model/hls_cache/headers.rs | 20 +- backend/src/api/model/hls_cache/ids.rs | 18 + backend/src/api/model/hls_cache/lease.rs | 39 +- backend/src/api/model/hls_cache/manager.rs | 98 +- .../api/model/hls_cache/manifest_commit.rs | 20 +- .../src/api/model/hls_cache/manifest_fetch.rs | 760 ++++++-- backend/src/api/model/hls_cache/map.rs | 2 +- .../src/api/model/hls_cache/map_fetcher.rs | 159 +- backend/src/api/model/hls_cache/mod.rs | 60 +- .../src/api/model/hls_cache/observability.rs | 131 +- backend/src/api/model/hls_cache/origin.rs | 38 +- backend/src/api/model/hls_cache/prefetch.rs | 21 +- backend/src/api/model/hls_cache/qos.rs | 36 +- backend/src/api/model/hls_cache/refresh.rs | 489 ++++- backend/src/api/model/hls_cache/renderer.rs | 34 +- .../src/api/model/hls_cache/resource_fetch.rs | 883 +++++++-- backend/src/api/model/hls_cache/response.rs | 95 +- .../api/model/hls_cache/segment_fetcher.rs | 361 +++- .../src/api/model/hls_cache/segment_repair.rs | 53 +- .../api/model/hls_cache/segment_watchdog.rs | 9 +- backend/src/api/model/hls_cache/session.rs | 10 +- backend/src/api/model/hls_cache/timeline.rs | 26 +- backend/src/api/model/hls_cache/transient.rs | 8 +- .../api/model/hls_cache/transient_fetcher.rs | 1444 ++++++++++++-- .../src/api/model/metadata_update_manager.rs | 1 + backend/src/api/model/mod.rs | 14 +- backend/src/api/model/model_utils.rs | 321 +++- backend/src/api/model/stream.rs | 54 +- backend/src/api/model/stream_error.rs | 3 + .../api/model/streams/active_client_stream.rs | 121 +- .../src/api/model/streams/provider_stream.rs | 46 +- .../model/streams/provider_stream_factory.rs | 1300 +++++++++++-- backend/src/api/panel_api.rs | 2 + .../parser/hls/transient_manifest.rs | 145 +- backend/src/processing/parser/m3u.rs | 1 + backend/src/processing/parser/xtream.rs | 14 +- backend/src/processing/processor/playlist.rs | 190 +- backend/src/repository/library_repository.rs | 2 + .../src/repository/m3u_playlist_iterator.rs | 1 + backend/src/repository/playlist_repository.rs | 2 + backend/src/repository/xtream_repository.rs | 1 + .../compression/compressed_file_reader.rs | 33 +- .../compressed_file_reader_async.rs | 29 +- .../utils/compression/compression_utils.rs | 38 +- backend/src/utils/db_viewer.rs | 1 + backend/src/utils/network/content_coding.rs | 917 +++++++++ backend/src/utils/network/mod.rs | 7 + backend/src/utils/network/request.rs | 1680 ++++++++++++----- backend/src/utils/network/xtream.rs | 1 + .../configuration/hls-cache-state-machine.md | 11 +- .../configuration/shared-hls-runtime-flow.md | 19 +- docs/src/configuration/shared-hls-sessions.md | 25 +- shared/src/model/playlist.rs | 201 +- shared/src/model/stream_info.rs | 4 + shared/src/model/stream_properties.rs | 1 + shared/src/utils/constants.rs | 4 - 64 files changed, 10183 insertions(+), 2729 deletions(-) create mode 100644 backend/src/utils/network/content_coding.rs diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ffc52fd57..3083eda54 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -73,7 +73,7 @@ dashmap.workspace = true hyper = "1.10.1" hyper-util = "0.1.20" socket2 = { version = "0.6.4", features = ["all"] } -async-compression = { version = "0.4.42", features = ["tokio", "gzip", "zlib"] } +async-compression = { version = "0.4.42", features = ["tokio", "brotli", "deflate", "gzip", "zlib", "zstd"] } chrono-tz = "0.10.4" zeroize.workspace = true uuid = { version = "1.23.3", features = ["v4"] } diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index c459fbbd4..a4b81d1d3 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -6,17 +6,17 @@ use crate::{ create_provider_connections_exhausted_stream, create_provider_stream, get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled, tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, - ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamFactoryOptions, ProviderStreamInfo, - ProviderStreamCustomReason, ProviderStreamState, SharedStreamManager, StreamDetails, StreamError, - StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, + ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamCustomReason, + ProviderStreamFactoryOptions, ProviderStreamInfo, ProviderStreamState, SharedStreamManager, StreamDetails, + StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, }, }, auth::Fingerprint, media_server::{ playback::{ media_server_image_response as open_media_server_proxy_image_response, - media_server_stream_response as open_media_server_proxy_stream_response, - parse_media_server_image_ref, parse_media_server_stream_ref, + media_server_stream_response as open_media_server_proxy_stream_response, parse_media_server_image_ref, + parse_media_server_stream_ref, }, plex::client::PlexCatalogClient, MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, @@ -43,16 +43,16 @@ use log::{debug, error, info, log_enabled, trace, warn}; use serde::Serialize; use shared::{ concat_string, + defaults::{DASH_EXT, HLS_EXT}, model::{ - Claims, InputFetchMethod, InputType, PlaylistEntry, PlaylistItemType, ProxyType, StreamChannel, StreamInfo, TargetType, - UserConnectionPermission, VirtualId, XtreamCluster, ConfigTargetOptions, + Claims, ConfigTargetOptions, InputFetchMethod, InputType, PlaylistEntry, PlaylistItemType, ProxyType, + StreamChannel, StreamInfo, TargetType, UserConnectionPermission, VirtualId, XtreamCluster, }, utils::{ - bin_serialize, current_time_secs, extract_extension_from_url, get_credentials_from_url, - human_readable_kbps, is_sanitize_sensitive_info_enabled, replace_url_extension, sanitize_sensitive_info, trim_slash, Internable, + bin_serialize, current_time_secs, extract_extension_from_url, get_credentials_from_url, human_readable_kbps, + is_sanitize_sensitive_info_enabled, replace_url_extension, sanitize_sensitive_info, trim_slash, Internable, CONTENT_TYPE_CBOR, CONTENT_TYPE_JSON, }, - defaults::{DASH_EXT, HLS_EXT,}, }; use smallvec::SmallVec; use std::{ @@ -64,8 +64,10 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; -use tokio::sync::RwLock; +use tokio::{ + io::{AsyncReadExt, AsyncSeekExt}, + sync::RwLock, +}; use tokio_util::io::ReaderStream; use url::Url; @@ -131,17 +133,14 @@ pub(crate) async fn resolve_playback_request_admission( prepare_only, terminate, }); - let limits_enabled = - (user.max_connections > 0 || user.soft_connections > 0) && app_state.app_config.config.load().user_access_control; + let limits_enabled = (user.max_connections > 0 || user.soft_connections > 0) + && app_state.app_config.config.load().user_access_control; // Handle explicit Terminate: run termination and return exhausted permission. // No admission strategies are evaluated — termination immediately expires the playback. if request_class == PlaybackRequestClass::Terminate { if let Some(session) = user_session { - app_state - .active_users - .terminate_session(&user.username, session.token.as_str()) - .await; + app_state.active_users.terminate_session(&user.username, session.token.as_str()).await; } return ( crate::api::model::ConnectionAdmission { @@ -173,8 +172,7 @@ pub(crate) async fn resolve_playback_request_admission( if request_class == PlaybackRequestClass::FollowUp || !limits_enabled { return ( crate::api::model::ConnectionAdmission { - permission: user_session - .map_or(UserConnectionPermission::Allowed, |session| session.permission), + permission: user_session.map_or(UserConnectionPermission::Allowed, |session| session.permission), kind: user_session .and_then(|session| session.connection_kind) .or(Some(crate::api::model::ConnectionKind::Normal)), @@ -196,7 +194,7 @@ pub(crate) async fn resolve_playback_request_admission( activate_unbound_session, eviction_reentry_guard, ) - .await; + .await; (result.admission, result.grace_mode, request_class) } @@ -475,12 +473,16 @@ macro_rules! try_result_not_found { }; } -use crate::api::panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}; -use crate::utils::LRUResourceCache; +use crate::{ + api::panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}, + utils::LRUResourceCache, +}; pub use internal_server_error; -use shared::error::TuliproxError; -use shared::model::{AdmissionStrategy, ConnectFailureReason, FailureStage, GeoIpUnavailablePolicy}; -use shared::defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}; +use shared::{ + defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, + error::TuliproxError, + model::{AdmissionStrategy, ConnectFailureReason, FailureStage, GeoIpUnavailablePolicy}, +}; pub use try_option_bad_request; pub use try_option_forbidden; pub use try_result_bad_request; @@ -740,6 +742,7 @@ pub struct ForceStreamRequestContext<'a> { pub input: &'a Arc, pub user: &'a ProxyUserCredentials, pub session_reservation_ttl_secs: u64, + pub(crate) content_representation: crate::api::model::ProviderContentRepresentationMode, } /// Constructs a `StreamOptions` object based on the application's reverse proxy configuration. @@ -879,7 +882,7 @@ where eviction_reentry_guard, &target.addr, ) - .await + .await { debug!( "Skipping eviction strategy {strategy:?} for recently evicted request of user {username} targeting {}", @@ -902,7 +905,7 @@ where session_token, activate_unbound_session, ) - .await; + .await; if retry_admission.permission == UserConnectionPermission::Allowed { return Some(AdmissionStrategyResolution { admission: retry_admission, @@ -951,7 +954,7 @@ pub(in crate::api) async fn resolve_admission_with_strategies( session_token, activate_unbound_session, ) - .await; + .await; if admission.permission != UserConnectionPermission::Exhausted { return AdmissionStrategyResolution { admission, grace_mode: None, grace_context: None }; @@ -988,7 +991,7 @@ pub(in crate::api) async fn resolve_admission_with_strategies( admission.kind, build_grace_ctx, ) - .await + .await { return resolution; } @@ -1042,10 +1045,8 @@ pub(in crate::api) async fn evaluate_remaining_strategies_after_grace( // admission.kind is used only inside build_grace_ctx for the Grace case's // GraceResolutionContext.kind. Both paths (helper early-return and caller // exhausted construction) use original_kind, so this is safe. - let admission = crate::api::model::ConnectionAdmission { - permission: UserConnectionPermission::Exhausted, - kind: original_kind, - }; + let admission = + crate::api::model::ConnectionAdmission { permission: UserConnectionPermission::Exhausted, kind: original_kind }; let build_grace_ctx = |global_idx: usize| GraceResolutionContext { strategy_index: global_idx, strategies: strategies.clone(), @@ -1071,7 +1072,7 @@ pub(in crate::api) async fn evaluate_remaining_strategies_after_grace( original_kind, build_grace_ctx, ) - .await + .await { return resolution; } @@ -1136,10 +1137,8 @@ async fn activate_session_before_stream_open( // If it is no longer counted, classify it from the current lifecycle so // stale FollowUp requests cannot bypass admission. // If it became counted, classify it so stale Activate requests don't double count. - let current_session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let current_session = + app_state.active_users.get_and_update_user_session(&user.username, session_token).await; classify_playback_request(PlaybackRequestFacts { item_type, existing_session: current_session.as_ref(), @@ -1150,10 +1149,7 @@ async fn activate_session_before_stream_open( request_class } } else { - let existing_session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let existing_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; classify_playback_request(PlaybackRequestFacts { item_type, existing_session: existing_session.as_ref(), @@ -1161,8 +1157,8 @@ async fn activate_session_before_stream_open( terminate: false, }) }; - let limits_enabled = - app_state.app_config.config.load().user_access_control && (user.max_connections > 0 || user.soft_connections > 0); + let limits_enabled = app_state.app_config.config.load().user_access_control + && (user.max_connections > 0 || user.soft_connections > 0); // Prepare: session setup without admission cost. The caller handles the actual activation. // FollowUp: already counted, no re-admission needed. // GracePeriod: grace already granted, no re-evaluation needed. @@ -1173,10 +1169,7 @@ async fn activate_session_before_stream_open( if connection_permission == UserConnectionPermission::GracePeriod { // Materialize grace lifecycle under the guard so the session state is consistent. // Determine which grace mode applies by checking the current session state. - let current_session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let current_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; let (_, resolved_grace) = match current_session.as_ref().map(|s| &s.lifecycle) { Some(crate::api::model::PlaybackLifecycle::PendingProvider { .. }) => { // Session already in PendingProvider — refresh deadline. @@ -1185,19 +1178,24 @@ async fn activate_session_before_stream_open( .active_users .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) .await; - (crate::api::model::PlaybackLifecycle::PendingProvider { - data: crate::api::model::PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), - deadline, - version: current_session.as_ref().map_or(0, |s| { - if let crate::api::model::PlaybackLifecycle::PendingProvider { data } = &s.lifecycle { - data.version - } else { 0 } - }), - wake_source: None, + ( + crate::api::model::PlaybackLifecycle::PendingProvider { + data: crate::api::model::PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline, + version: current_session.as_ref().map_or(0, |s| { + if let crate::api::model::PlaybackLifecycle::PendingProvider { data } = &s.lifecycle { + data.version + } else { + 0 + } + }), + wake_source: None, + }, }, - }, Some(crate::api::model::GraceMode::Hold)) + Some(crate::api::model::GraceMode::Hold), + ) } Some(crate::api::model::PlaybackLifecycle::GraceActive) => { // Already in GraceActive — infer mode from item_type. @@ -1215,17 +1213,25 @@ async fn activate_session_before_stream_open( let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); let _ = app_state .active_users - .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) - .await; - (crate::api::model::PlaybackLifecycle::PendingProvider { - data: crate::api::model::PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), + .mark_pending_provider( + &user.username, + session_token, + PendingProviderReason::GraceHold, deadline, - version: 1, - wake_source: None, + ) + .await; + ( + crate::api::model::PlaybackLifecycle::PendingProvider { + data: crate::api::model::PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline, + version: 1, + wake_source: None, + }, }, - }, Some(crate::api::model::GraceMode::Hold)) + Some(crate::api::model::GraceMode::Hold), + ) } else { app_state.active_users.mark_grace_active(&user.username, session_token).await; (crate::api::model::PlaybackLifecycle::GraceActive, Some(crate::api::model::GraceMode::Instant)) @@ -1258,20 +1264,22 @@ async fn activate_session_before_stream_open( }; } - let placeholder_transition_version = Some(app_state - .active_users - .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id, - provider: input.name.as_ref(), - stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await); + let placeholder_transition_version = Some( + app_state + .active_users + .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id, + provider: input.name.as_ref(), + stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await, + ); let result = resolve_admission_with_strategies( app_state, @@ -1289,7 +1297,7 @@ async fn activate_session_before_stream_open( EvictionReentryGuard::Session(session_token) }, ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; let grace_context = result.grace_context; @@ -1305,22 +1313,18 @@ async fn activate_session_before_stream_open( } else if matches!(grace_mode, Some(crate::api::model::GraceMode::Instant)) { // Instant: session is provisionally active immediately. Counts against admission limits // until the grace window resolves (success -> Active, failure -> Expired). - app_state - .active_users - .mark_grace_active(&user.username, session_token) - .await; + app_state.active_users.mark_grace_active(&user.username, session_token).await; } } - PlaybackActivationResult { - admission, - grace_mode, - grace_context, - placeholder_transition_version, - } + PlaybackActivationResult { admission, grace_mode, grace_context, placeholder_transition_version } } -pub fn get_stream_alternative_url(stream_url: &str, input: &ConfigInput, alias_input: &Arc) -> Option { +pub fn get_stream_alternative_url( + stream_url: &str, + input: &ConfigInput, + alias_input: &Arc, +) -> Option { if input.input_type.is_m3u() && input.get_matched_config_by_url(stream_url).is_none() { return get_stream_alternative_url_m3u(stream_url, input, alias_input); } @@ -1404,8 +1408,8 @@ fn providerless_m3u_url_has_explicit_credentials(url: &Url) -> bool { !url.username().is_empty() || url.password().is_some() || url - .query_pairs() - .any(|(key, _)| key.eq_ignore_ascii_case("username") || key.eq_ignore_ascii_case("password")) + .query_pairs() + .any(|(key, _)| key.eq_ignore_ascii_case("username") || key.eq_ignore_ascii_case("password")) } /// Look for an account signature in the stream URL that matches the input @@ -1433,12 +1437,9 @@ fn find_input_account_by_signature( // parent input for all aliases — see ConfigInputAlias definition. if let Some(aliases) = input.aliases.as_ref() { for alias in aliases { - if let Some(user_info) = InputUserInfo::new( - input.input_type, - alias.username.as_deref(), - alias.password.as_deref(), - &alias.url, - ) { + if let Some(user_info) = + InputUserInfo::new(input.input_type, alias.username.as_deref(), alias.password.as_deref(), &alias.url) + { if stream_url_account_matches(stream_url, &user_info) { return Some((alias.url.clone(), Some(user_info.username), Some(user_info.password))); } @@ -1493,8 +1494,7 @@ fn rewrite_query_auth_fields(url: &mut Url, new_username: &str, new_password: &s } fn collect_path_segments(url: &Url) -> Option> { - url.path_segments() - .map(|segments| segments.map(ToOwned::to_owned).collect::>()) + url.path_segments().map(|segments| segments.map(ToOwned::to_owned).collect::>()) } fn find_path_auth_segment_index(segments: &[String], username: &str, password: &str) -> Option { @@ -1551,9 +1551,9 @@ fn stream_url_matches_provider(stream_url: &str, provider_cfg: &ProviderConfig) } fn stream_url_base_matches(stream_url: &str, base_url: &str) -> bool { - stream_url.strip_prefix(base_url).is_some_and(|remaining| { - remaining.is_empty() || remaining.starts_with(['/', '?', '#']) - }) + stream_url + .strip_prefix(base_url) + .is_some_and(|remaining| remaining.is_empty() || remaining.starts_with(['/', '?', '#'])) } fn stream_url_account_matches(stream_url: &str, user_info: &crate::model::InputUserInfo) -> bool { @@ -1771,25 +1771,20 @@ async fn resolve_streaming_strategy( // panel_api provisioning/loading is handled later in the stream creation flow let mut release_failed_mapping = false; - let stream_response_params = if let Some(allocation) = provider_connection_handle.as_ref().map(|ph| &ph.allocation) { + let stream_response_params = if let Some(allocation) = provider_connection_handle.as_ref().map(|ph| &ph.allocation) + { match allocation { ProviderAllocation::Exhausted => { debug!("Provider {} is exhausted. No connections allowed.", input.name); let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom { - response: stream, - reason: ProviderStreamCustomReason::ProviderExhausted, - } + ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } } ProviderAllocation::Available(ref provider_cfg) | ProviderAllocation::GracePeriod(ref provider_cfg) => { // Keep the URL only when it already targets the selected provider account. Hot reload can leave old // alias URLs in persisted playlists until the next processing run. - if let Some((selected_provider_name, url)) = select_provider_stream_url( - stream_url, - input, - provider_cfg, - accept_requested_stream_url, - ) { + if let Some((selected_provider_name, url)) = + select_provider_stream_url(stream_url, input, provider_cfg, accept_requested_stream_url) + { debug_if_enabled!( "provider session: input={} provider_cfg={} user={} allocation={} stream_url={}", sanitize_sensitive_info(&input.name), @@ -1822,10 +1817,7 @@ async fn resolve_streaming_strategy( } else { debug!("Provider {} is exhausted. No connections allowed.", input.name); let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom { - response: stream, - reason: ProviderStreamCustomReason::ProviderExhausted, - } + ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } }; if release_failed_mapping { @@ -1851,10 +1843,10 @@ fn get_grace_period_millis( ) -> u64 { if config_grace_period_millis > 0 && ( - matches!(stream_response_params, ProviderStreamState::GracePeriod(_, _)) // provider grace period + matches!(stream_response_params, ProviderStreamState::GracePeriod(_, _)) // provider grace period || connection_permission == UserConnectionPermission::GracePeriod - // user grace period - ) + // user grace period + ) { config_grace_period_millis } else { @@ -1891,6 +1883,7 @@ async fn create_stream_response_details( input: &Arc, stream_channel: &StreamChannel, item_type: PlaylistItemType, + content_representation: crate::api::model::ProviderContentRepresentationMode, share_stream: bool, connection_permission: UserConnectionPermission, force_provider: Option<&Arc>, @@ -1921,7 +1914,7 @@ async fn create_stream_response_details( accept_requested_stream_url, }, ) - .await; + .await; let mut grace_period_options = app_state.get_grace_options(); grace_period_options.period_millis = get_grace_period_millis( connection_permission, @@ -1939,12 +1932,8 @@ async fn create_stream_response_details( if matches!( streaming_strategy.provider_stream_state, - ProviderStreamState::Custom { - reason: ProviderStreamCustomReason::ProviderExhausted, - .. - } - ) - && can_provision_on_exhausted(app_state, input) + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } + ) && can_provision_on_exhausted(app_state, input) { if let Some(handle) = streaming_strategy.provider_handle.take() { app_state.connection_manager.release_provider_handle(Some(handle)).await; @@ -1953,22 +1942,21 @@ async fn create_stream_response_details( "panel_api: provider connections exhausted; sending provisioning stream for input {}", sanitize_sensitive_info(&input.name) ); - return Ok(create_panel_api_provisioning_stream_details( + let mut details = create_panel_api_provisioning_stream_details( app_state, input, guard_provider_name.clone().or_else(|| Some(input.name.clone())), &grace_period_options, fingerprint.addr, virtual_id, - )); + ); + details.content_representation = content_representation; + return Ok(details); } match streaming_strategy.provider_stream_state { // custom stream means we display our own stream like connection exhausted, channel-unavailable... - ProviderStreamState::Custom { - response: provider_stream, - .. - } => { + ProviderStreamState::Custom { response: provider_stream, .. } => { let (stream, stream_info) = provider_stream; // When allocation is exhausted or no connection was acquired, guard_provider_name is None. // Use input.name as fallback so the provider field is never empty. @@ -1984,6 +1972,7 @@ async fn create_stream_response_details( disable_provider_grace: false, reconnect_flag: None, provider_handle: streaming_strategy.provider_handle.clone(), + content_representation, grace_resolution_context, }) } @@ -2043,6 +2032,7 @@ async fn create_stream_response_details( client_ip: Some(&fingerprint.client_ip), stream_channel: Some(stream_channel), connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation, }); let provider_config = input.get_resolve_provider(url.as_ref()); @@ -2054,7 +2044,7 @@ async fn create_stream_response_details( &app_state.http_client.load(), provider_stream_factory_options, ) - .await + .await { None => (None, None), Some((stream, info)) => (Some(stream), info), @@ -2077,15 +2067,12 @@ async fn create_stream_response_details( } } - // If no upstream stream is ready, release the provider. - // Even if provider grace intentionally deferred the open, we must release the handle - // here because the deferred open context will acquire a fresh slot when it resumes. - let provider_handle = if stream.is_none() { + // An intentional deferred open must retain its grace allocation until body polling + // resumes the provider request. Other failed opens release their allocation here. + let provider_handle = if stream.is_none() && !defer_provider_stream_until_grace_check { let provider_handle = streaming_strategy.provider_handle.take(); app_state.connection_manager.release_provider_handle(provider_handle).await; - if !defer_provider_stream_until_grace_check { - error!("Can't open stream {}", sanitize_sensitive_info(&request_url)); - } + error!("Can't open stream {}", sanitize_sensitive_info(&request_url)); None } else { streaming_strategy.provider_handle.take() @@ -2102,6 +2089,7 @@ async fn create_stream_response_details( disable_provider_grace: false, reconnect_flag, provider_handle, + content_representation, grace_resolution_context, }) } @@ -2129,9 +2117,10 @@ where P: PlaylistEntry, { pub fn get_query_path(&self, provider_id: u32, url: &str) -> String { - let extension = self - .stream_ext - .map_or_else(|| extract_extension_from_url(url).map_or_else(String::new, ToString::to_string), ToString::to_string); + let extension = self.stream_ext.map_or_else( + || extract_extension_from_url(url).map_or_else(String::new, ToString::to_string), + ToString::to_string, + ); // if there is an action_path (like for timeshift duration/start), it will be added in front of the stream_id if self.action_path.is_empty() { @@ -2266,7 +2255,7 @@ fn is_throttled_stream(item_type: PlaylistItemType, throttle_kbps: usize) -> boo fn prepare_body_stream(app_state: &Arc, item_type: PlaylistItemType, stream: S) -> axum::body::Body where - S: futures::Stream> + Send + 'static, + S: futures::Stream> + Send + 'static, { let throttle_kbps = usize::try_from(get_stream_throttle(app_state)).unwrap_or_default(); let body_stream = if is_throttled_stream(item_type, throttle_kbps) { @@ -2298,7 +2287,12 @@ async fn open_media_server_stream_for_input( .provider("media-server") .detail("media-server playback proxy is not implemented for this input type")); } - InputType::M3u | InputType::Xtream | InputType::M3uBatch | InputType::XtreamBatch | InputType::Library | InputType::Staged => { + InputType::M3u + | InputType::Xtream + | InputType::M3uBatch + | InputType::XtreamBatch + | InputType::Library + | InputType::Staged => { return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) .provider("media-server") .detail("playlist item is not backed by a media-server input")); @@ -2359,10 +2353,8 @@ pub async fn force_provider_stream_response( ctx: ForceStreamRequestContext<'_>, grace_mode: Option, ) -> impl IntoResponse + Send { - let _transition_guard = app_state - .active_users - .acquire_playback_transition(&ctx.user.username, &user_session.token) - .await; + let _transition_guard = + app_state.active_users.acquire_playback_transition(&ctx.user.username, &user_session.token).await; let stream_options = get_stream_options(app_state); let share_stream = false; let connection_permission = UserConnectionPermission::Allowed; @@ -2415,6 +2407,7 @@ pub async fn force_provider_stream_response( ctx.input, &stream_channel, item_type, + ctx.content_representation, share_stream, connection_permission, preferred_provider, @@ -2430,7 +2423,7 @@ pub async fn force_provider_stream_response( grace_mode.map(|mode| matches!(mode, crate::api::model::GraceMode::Hold)), None, ) - .await + .await { Ok(stream_details) => stream_details, Err(err) => { @@ -2453,7 +2446,7 @@ pub async fn force_provider_stream_response( stream_details.stream.is_some(), stream_details.has_deferred_provider_open(), ) - .await; + .await; let provider_response = stream_details.stream_info.as_ref().map(|(h, sc, url, cvt)| (h.clone(), *sc, url.clone(), *cvt)); if ctx.session_reservation_ttl_secs > 0 { @@ -2481,7 +2474,7 @@ pub async fn force_provider_stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await; let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); let mut response = axum::response::Response::builder().status(status_code); @@ -2543,10 +2536,7 @@ pub(crate) async fn stream_response( allow_exhausted_shared_reconnect: bool, grace_mode: Option, ) -> impl IntoResponse + Send { - let _transition_guard = app_state - .active_users - .acquire_playback_transition(&user.username, session_token) - .await; + let _transition_guard = app_state.active_users.acquire_playback_transition(&user.username, session_token).await; let request_log_stream_url = resolve_request_url_for_logging(input, stream_url); if log_enabled!(log::Level::Trace) { trace!("Try to open stream {}", sanitize_sensitive_info(request_log_stream_url.as_ref())); @@ -2574,7 +2564,7 @@ pub(crate) async fn stream_response( socket_bound, }, ) - .await; + .await; let grace_mode = activation.grace_mode.or(grace_mode); connection_permission = activation.admission.permission; connection_kind = activation.admission.kind.unwrap_or(connection_kind); @@ -2598,7 +2588,7 @@ pub(crate) async fn stream_response( session_token, req_headers, ) - .await + .await { return value.into_response(); } @@ -2619,7 +2609,7 @@ pub(crate) async fn stream_response( session_token, req_headers, ) - .await + .await { debug_if_enabled!("Opportunistic shared stream reuse for {}", sanitize_sensitive_info(stream_url)); return value.into_response(); @@ -2653,14 +2643,11 @@ pub(crate) async fn stream_response( &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) - .into_response(); + .into_response(); } let stream_options = get_stream_options(app_state); - let session_state = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; let mut stream_details = match create_stream_response_details( app_state, &stream_options, @@ -2671,6 +2658,7 @@ pub(crate) async fn stream_response( input, &stream_channel, item_type, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, share_stream, connection_permission, pinned_provider, @@ -2686,7 +2674,7 @@ pub(crate) async fn stream_response( grace_mode.map(|m| matches!(m, crate::api::model::GraceMode::Hold)), activation.grace_context.clone(), ) - .await + .await { Ok(stream_details) => stream_details, Err(err) => { @@ -2740,7 +2728,7 @@ pub(crate) async fn stream_response( stream_details.stream.is_some(), stream_details.has_deferred_provider_open(), ) - .await; + .await; // Captured before `stream_details` is moved into `create_active_client_stream`. // The pinning rule is centralized in `should_pin_provider_for_session` so it stays @@ -2781,7 +2769,7 @@ pub(crate) async fn stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await; let stream_resp = if is_stream_shared { debug_if_enabled!( "Streaming shared stream request from {}", @@ -2800,7 +2788,7 @@ pub(crate) async fn stream_response( connection_priority_for_kind(user, connection_kind), connection_kind, ) - .await + .await { let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); @@ -2960,10 +2948,7 @@ async fn prepare_stream_metering( .shared_stream_manager .get_or_register_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) .await; - return StreamMeteringConfig { - meter_uid, - meter_stream: has_stream || has_deferred_provider_open, - }; + return StreamMeteringConfig { meter_uid, meter_stream: has_stream || has_deferred_provider_open }; } else if has_stream || has_deferred_provider_open { let meter_uid = app_state.connection_manager.next_stream_uid(); return StreamMeteringConfig { meter_uid, meter_stream: true }; @@ -3062,7 +3047,7 @@ async fn try_shared_stream_response_if_any( connection_priority_for_kind(user, connection_kind), connection_kind, ) - .await + .await { debug_if_enabled!("Using shared stream {}", sanitize_sensitive_info(stream_url)); if let Some(headers) = app_state.shared_stream_manager.get_shared_state_headers(stream_url).await { @@ -3074,7 +3059,8 @@ async fn try_shared_stream_response_if_any( let mut stream_details = StreamDetails::from_stream(stream, grace_period_options); stream_details.provider_name = provider; - let socket_bound = is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); + let socket_bound = + is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); if let Some(provider_name) = stream_details.provider_name.as_deref() { let _ = app_state .active_users @@ -3098,10 +3084,7 @@ async fn try_shared_stream_response_if_any( .get_or_register_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) .await; stream_channel.shared_stream_id = Some(u64::from(meter_uid)); - let metering = StreamMeteringConfig { - meter_uid, - meter_stream: false, - }; + let metering = StreamMeteringConfig { meter_uid, meter_stream: false }; let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { stream_details, app_state, @@ -3116,8 +3099,8 @@ async fn try_shared_stream_response_if_any( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await - .boxed(); + .await + .boxed(); let mut response = axum::response::Response::builder().status(status_code); for (key, value) in &header_map { response = response.header(key, value); @@ -3146,12 +3129,7 @@ pub(crate) async fn local_stream_response( check_path: bool, ) -> impl IntoResponse + Send { let _transition_guard = if let Some(session_token) = playback_session_token { - Some( - app_state - .active_users - .acquire_playback_transition(&user.username, session_token) - .await, - ) + Some(app_state.active_users.acquire_playback_transition(&user.username, session_token).await) } else { None }; @@ -3165,15 +3143,15 @@ pub(crate) async fn local_stream_response( let allow_session_reopen = if let Some(session_token) = playback_session_token { user.max_connections > 0 && app_state - .active_users - .connection_permission_for_session( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await - != UserConnectionPermission::Exhausted + .active_users + .connection_permission_for_session( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await + != UserConnectionPermission::Exhausted } else { false }; @@ -3193,7 +3171,7 @@ pub(crate) async fn local_stream_response( &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) - .into_response(); + .into_response(); } connection_permission = UserConnectionPermission::Allowed; } @@ -3214,7 +3192,7 @@ pub(crate) async fn local_stream_response( if check_path { let Ok(canonical_metadata) = tokio::fs::metadata(&canonical).await else { return internal_server_error!() }; - + #[cfg(unix)] { use std::os::unix::fs::MetadataExt; @@ -3311,7 +3289,7 @@ pub(crate) async fn local_stream_response( socket_bound, }, ) - .await; + .await; grace_mode = activation.grace_mode; connection_permission = activation.admission.permission; connection_kind = activation.admission.kind.unwrap_or(connection_kind); @@ -3331,7 +3309,7 @@ pub(crate) async fn local_stream_response( &fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted, ) - .into_response(); + .into_response(); } } let mut grace_period_options = app_state.get_grace_options(); @@ -3382,7 +3360,7 @@ pub(crate) async fn local_stream_response( meter_uid: metering.meter_uid, meter_stream: metering.meter_stream, }) - .await; + .await; let mut response = Response::new(axum::body::Body::from_stream(stream)); @@ -3483,19 +3461,7 @@ async fn build_resource_stream_response( let mime_type = get_mime_type(response.headers(), resource_url); let has_content_range = response.headers().contains_key(header::CONTENT_RANGE); for (key, value) in response.headers() { - let name = key.as_str(); - let is_hop_by_hop = matches!( - name.to_ascii_lowercase().as_str(), - "connection" - | "keep-alive" - | "proxy-authenticate" - | "proxy-authorization" - | "te" - | "trailer" - | "transfer-encoding" - | "upgrade" - ); - if !is_hop_by_hop { + if !is_hop_by_hop_response_header(key) { response_builder = response_builder.header(key, value); } } @@ -3564,7 +3530,7 @@ async fn fetch_resource_with_retry( default_user_agent.as_deref(), ) }) - .await + .await else { return None; }; @@ -3580,7 +3546,9 @@ async fn fetch_resource_with_retry( let mut response_builder = axum::response::Response::builder().status(status); for (key, value) in response.headers() { - response_builder = response_builder.header(key, value); + if !is_hop_by_hop_response_header(key) { + response_builder = response_builder.header(key, value); + } } let stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); @@ -3627,8 +3595,8 @@ pub async fn resource_response( mime_type.unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM.to_string()), Some("public, max-age=14400"), ) - .await - .into_response(); + .await + .into_response(); } } trace_if_enabled!("Try to fetch resource {}", sanitize_sensitive_info(resource_url)); @@ -3666,7 +3634,12 @@ async fn open_media_server_image_resource( .provider("media-server") .detail("media-server image proxy is not implemented for this input type")); } - InputType::M3u | InputType::Xtream | InputType::M3uBatch | InputType::XtreamBatch | InputType::Library | InputType::Staged => { + InputType::M3u + | InputType::Xtream + | InputType::M3uBatch + | InputType::XtreamBatch + | InputType::Library + | InputType::Staged => { return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) .provider("media-server") .detail("media-server image input is not backed by a media-server input")); @@ -3800,7 +3773,7 @@ pub fn json_or_bin_response(accept: Option<&str>, data: &T) -> imp pub fn stream_json_or_bin_response

( accept: Option<&str>, - data: Box + Send>, + data: Box + Send>, ) -> axum::response::Response where P: serde::Serialize + Send + 'static, @@ -3814,7 +3787,7 @@ where pub fn stream_json_or_bin_response_stream(accept: Option<&str>, data: S) -> axum::response::Response where P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, + S: Stream + Send + Unpin + 'static, { if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { return stream_bin_array_stream(data); @@ -3902,13 +3875,13 @@ pub(crate) fn should_allow_exhausted_shared_reconnect( ) -> bool { share_stream && user_session.is_some_and(|session| { - session.permission != UserConnectionPermission::Exhausted - && session.virtual_id == requested_virtual_id - && session.stream_url.as_ref() == requested_stream_url - }) + session.permission != UserConnectionPermission::Exhausted + && session.virtual_id == requested_virtual_id + && session.stream_url.as_ref() == requested_stream_url + }) } -pub fn stream_json_array

(iter: Box + Send>) -> axum::response::Response +pub fn stream_json_array

(iter: Box + Send>) -> axum::response::Response where P: serde::Serialize + Send + 'static, { @@ -3936,7 +3909,7 @@ where try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) } -pub fn stream_bin_array

(iter: Box + Send>) -> axum::response::Response +pub fn stream_bin_array

(iter: Box + Send>) -> axum::response::Response where P: serde::Serialize + Send + 'static, { @@ -3960,11 +3933,11 @@ where // CBOR: start indefinite-length array Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) }) - .chain(stream) - .chain(stream::once(async { - // CBOR: end indefinite-length array - Ok::<_, Infallible>(Bytes::from_static(&[0xff])) - })), + .chain(stream) + .chain(stream::once(async { + // CBOR: end indefinite-length array + Ok::<_, Infallible>(Bytes::from_static(&[0xff])) + })), ); try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) @@ -3973,7 +3946,7 @@ where pub fn stream_json_array_stream(stream: S) -> axum::response::Response where P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, + S: Stream + Send + Unpin + 'static, { let stream = stream::unfold((stream, true), |(mut stream, first)| async move { match stream.next().await { @@ -4002,7 +3975,7 @@ where pub fn stream_bin_array_stream(stream: S) -> axum::response::Response where P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, + S: Stream + Send + Unpin + 'static, { let stream = stream::unfold(stream, |mut stream| async move { match stream.next().await { @@ -4075,7 +4048,6 @@ pub fn empty_json_response_as_array() -> axum::http::Result Arc { test_runtime_provider_with_type(url, username, password, InputType::Xtream) @@ -4136,12 +4114,8 @@ mod tests { } fn test_runtime_provider_without_credentials(url: &str, input_type: InputType) -> Arc { - let input = ConfigInput { - name: "provider".intern(), - url: url.to_string(), - input_type, - ..ConfigInput::default() - }; + let input = + ConfigInput { name: "provider".intern(), url: url.to_string(), input_type, ..ConfigInput::default() }; Arc::new(RuntimeProviderConfig::new( &input, Arc::new(RwLock::new(ProviderConfigConnection::default())), @@ -4223,8 +4197,7 @@ mod tests { ..ConfigInput::default() }; let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = - "http://source.example/player?token=source-user&username=source-user&password=source-pass"; + let stream_url = "http://source.example/player?token=source-user&username=source-user&password=source-pass"; let rewritten = get_stream_alternative_url(stream_url, &input, &alias); @@ -4238,10 +4211,7 @@ mod tests { fn stream_url_matches_provider_requires_base_url_and_account_identity() { let provider = test_runtime_provider("http://same.example", "selected-user", "selected-pass"); - assert!(stream_url_matches_provider( - "http://same.example/live/selected-user/selected-pass/123.ts", - &provider - )); + assert!(stream_url_matches_provider("http://same.example/live/selected-user/selected-pass/123.ts", &provider)); assert!(stream_url_matches_provider( "http://same.example/timeshift/selected-user/selected-pass/30/2026-06-15:20-00/123.ts", &provider @@ -4250,10 +4220,7 @@ mod tests { "http://same.example/future-route/selected-user/selected-pass/opaque/123.ts", &provider )); - assert!(!stream_url_matches_provider( - "http://same.example/live/other-user/other-pass/123.ts", - &provider - )); + assert!(!stream_url_matches_provider("http://same.example/live/other-user/other-pass/123.ts", &provider)); assert!(!stream_url_matches_provider( "http://same.example/timeshift/other-user/other-pass/30/2026-06-15:20-00/123.ts", &provider @@ -4287,10 +4254,7 @@ mod tests { fn stream_url_matches_provider_rejects_external_cdn_url_with_wrong_account_signature() { let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); - assert!(!stream_url_matches_provider( - "http://cdn.example/live/other-user/other-pass/123.ts", - &provider - )); + assert!(!stream_url_matches_provider("http://cdn.example/live/other-user/other-pass/123.ts", &provider)); assert!(!stream_url_matches_provider( "http://cdn.example/segment.ts?username=other-user&password=other-pass", &provider @@ -4325,10 +4289,7 @@ mod tests { ); // Matching path credentials -> allowed (account matches). - assert!(stream_url_matches_provider( - "http://cdn.example/live/selected-user/selected-pass/123.ts", - &provider - )); + assert!(stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); } #[test] @@ -4343,10 +4304,7 @@ mod tests { fn stream_url_matches_provider_rejects_external_cdn_url_for_xtream_even_with_valid_account_signature() { let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); - assert!(!stream_url_matches_provider( - "http://cdn.example/live/selected-user/selected-pass/123.ts", - &provider - )); + assert!(!stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); assert!(!stream_url_matches_provider( "http://cdn.example/segment.ts?username=selected-user&password=selected-pass", &provider @@ -4375,36 +4333,21 @@ mod tests { ..ConfigInput::default() }; - let main = find_input_account_by_signature( - "http://cdn.example/live/main-user/main-pass/1.ts", - &input, - ); + let main = find_input_account_by_signature("http://cdn.example/live/main-user/main-pass/1.ts", &input); assert_eq!( main, - Some(( - "http://provider.example".to_string(), - Some("main-user".to_string()), - Some("main-pass".to_string()), - )) + Some( + ("http://provider.example".to_string(), Some("main-user".to_string()), Some("main-pass".to_string()),) + ) ); - let alias = find_input_account_by_signature( - "http://cdn.example/live/alias-user/alias-pass/1.ts", - &input, - ); + let alias = find_input_account_by_signature("http://cdn.example/live/alias-user/alias-pass/1.ts", &input); assert_eq!( alias, - Some(( - "http://alias.example".to_string(), - Some("alias-user".to_string()), - Some("alias-pass".to_string()), - )) + Some(("http://alias.example".to_string(), Some("alias-user".to_string()), Some("alias-pass".to_string()),)) ); - assert_eq!( - find_input_account_by_signature("http://cdn.example/live/other/other/1.ts", &input), - None - ); + assert_eq!(find_input_account_by_signature("http://cdn.example/live/other/other/1.ts", &input), None); assert_eq!(find_input_account_by_signature("http://cdn.example/open/playlist.m3u8", &input), None); } @@ -4422,10 +4365,7 @@ mod tests { let stream_url = "http://cdn.example/live/source-user/source-pass/123.ts"; let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - assert_eq!( - rewritten, - Some("http://cdn.example/live/alias-user/alias-pass/123.ts".to_string()) - ); + assert_eq!(rewritten, Some("http://cdn.example/live/alias-user/alias-pass/123.ts".to_string())); } #[test] @@ -4508,7 +4448,8 @@ mod tests { input_type: InputType::M3u, ..ConfigInput::default() }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + let provider = + test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); let stream_url = "http://s.only4.tv/17113/video.m3u8?token=abc"; assert_eq!(get_stream_alternative_url(stream_url, &input, &provider), Some(stream_url.to_string())); @@ -4522,7 +4463,8 @@ mod tests { input_type: InputType::M3u, ..ConfigInput::default() }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + let provider = + test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); assert_eq!( get_stream_alternative_url( @@ -4542,7 +4484,8 @@ mod tests { input_type: InputType::M3u, ..ConfigInput::default() }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + let provider = + test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); assert_eq!(get_stream_alternative_url("http://user:pass@cdn.example/segment.ts", &input, &provider), None); } @@ -4620,18 +4563,9 @@ mod tests { #[test] fn media_server_playback_urls_are_proxy_only_redirect_guard_candidates() { - let plex_input = ConfigInput { - input_type: InputType::Plex, - ..ConfigInput::default() - }; - let emby_input = ConfigInput { - input_type: InputType::Emby, - ..ConfigInput::default() - }; - let m3u_input = ConfigInput { - input_type: InputType::M3u, - ..ConfigInput::default() - }; + let plex_input = ConfigInput { input_type: InputType::Plex, ..ConfigInput::default() }; + let emby_input = ConfigInput { input_type: InputType::Emby, ..ConfigInput::default() }; + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; assert!(is_media_server_playback_url( &plex_input, @@ -4653,7 +4587,7 @@ mod tests { &plex_input, "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" ) - .as_ref(), + .as_ref(), "media-server://" ); } @@ -4666,6 +4600,154 @@ mod tests { assert!(!should_compress_response(&response)); } + async fn spawn_legacy_hls_test_origin( + response_head: String, + response_body: Vec, + ) -> (SocketAddr, tokio::task::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("test origin accepts request"); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + socket.write_all(response_head.as_bytes()).await.expect("test origin writes response headers"); + socket.write_all(&response_body).await.expect("test origin writes response body"); + String::from_utf8_lossy(&request).into_owned() + }); + (origin_addr, origin_task) + } + + async fn forced_legacy_hls_test_response( + origin_addr: SocketAddr, + request_headers: &HeaderMap, + client_port: u16, + ) -> axum::response::Response { + let origin_url = format!("http://{origin_addr}/segment.ts"); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + ..ConfigInput::default() + }); + let app_config = create_test_provider_app_config(); + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let client_addr = SocketAddr::from(([127, 0, 0, 1], client_port)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + let session = UserSession { + token: format!("legacy-hls-marker-{client_port}"), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: origin_url.as_str().intern(), + provider_session_headers: HashMap::new(), + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + }; + let mut stream_channel = create_test_local_channel(&origin_url); + stream_channel.provider_id = u32::from(input.id); + stream_channel.input_name = Arc::clone(&input.name); + stream_channel.item_type = PlaylistItemType::Catchup; + stream_channel.cluster = XtreamCluster::Live; + stream_channel.url = origin_url.as_str().intern(); + + force_provider_stream_response( + &fingerprint, + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: request_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response() + } + + #[tokio::test] + async fn forced_hls_provider_response_disables_compression_and_streams_identity_bytes() { + const IDENTITY_BODY: &[u8] = b"legacy hls identity segment"; + + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(IDENTITY_BODY).await.expect("gzip test body encodes"); + encoder.shutdown().await.expect("gzip test encoder finishes"); + let encoded_body = encoder.into_inner(); + + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded_body.len() + ); + let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, encoded_body).await; + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_310).await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!should_compress_response(&response)); + assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + let body = response.into_body().collect().await.expect("legacy HLS response body").to_bytes(); + assert_eq!(body.as_ref(), IDENTITY_BODY); + + let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); + } + + #[tokio::test] + async fn forced_hls_unencoded_partial_response_preserves_range_and_disables_compression() { + const PARTIAL_BODY: &[u8] = b"cdef"; + let response_head = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes 2-5/10\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + PARTIAL_BODY.len() + ); + let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, PARTIAL_BODY.to_vec()).await; + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=2-")); + + let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_311).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert!(!should_compress_response(&response)); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); + assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); + let body = response.into_body().collect().await.expect("legacy HLS partial body").to_bytes(); + assert_eq!(body.as_ref(), PARTIAL_BODY); + + let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); + assert!(request.contains("\r\nrange: bytes=2-\r\n")); + } + #[test] fn test_regular_response_keeps_compression_enabled() { let response = Response::new(()); @@ -4753,15 +4835,12 @@ mod tests { accept_requested_stream_url: false, }, ) - .await; + .await; assert!(strict.provider_handle.is_none(), "strict provider affinity should not allocate a different provider"); assert!( matches!( strict.provider_stream_state, - ProviderStreamState::Custom { - reason: ProviderStreamCustomReason::ProviderExhausted, - .. - } + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } ), "strict provider affinity should fail closed when the pinned provider is unavailable" ); @@ -4781,7 +4860,7 @@ mod tests { accept_requested_stream_url: false, }, ) - .await; + .await; let (ProviderStreamState::Available(Some(fallback_provider), _) | ProviderStreamState::GracePeriod(Some(fallback_provider), _)) = fallback.provider_stream_state else { @@ -4822,7 +4901,7 @@ mod tests { accept_requested_stream_url: false, }, ) - .await; + .await; let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { panic!("request should allocate the main provider") @@ -4861,15 +4940,12 @@ mod tests { accept_requested_stream_url: false, }, ) - .await; + .await; assert!(strategy.provider_handle.is_none()); assert!(matches!( strategy.provider_stream_state, - ProviderStreamState::Custom { - reason: ProviderStreamCustomReason::UnmappedProviderUrl, - .. - } + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } )); app_state.active_provider.release_connection(&addr).await; @@ -5079,9 +5155,7 @@ mod tests { } } - fn create_test_app_state() -> Arc { - create_test_app_state_for_config(Arc::new(create_test_app_config())) - } + fn create_test_app_state() -> Arc { create_test_app_state_for_config(Arc::new(create_test_app_config())) } #[tokio::test] async fn create_api_proxy_user_defaults_output_clusters_to_all() { @@ -5278,11 +5352,8 @@ mod tests { #[test] fn classify_playback_request_marks_counted_session_as_follow_up() { - let session = create_test_session( - "tok-active", - PlaylistItemType::LiveHls, - crate::api::model::PlaybackLifecycle::Active, - ); + let session = + create_test_session("tok-active", PlaylistItemType::LiveHls, crate::api::model::PlaybackLifecycle::Active); let request_class = classify_playback_request(PlaybackRequestFacts { item_type: PlaylistItemType::LiveHls, @@ -5311,7 +5382,7 @@ mod tests { deadline: 30, version: 1, wake_source: None, - } + }, }, ); @@ -5482,7 +5553,7 @@ mod tests { socket_bound: true, }, ) - .await; + .await; assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); assert_eq!(activation.admission.kind, Some(crate::api::model::ConnectionKind::Normal)); @@ -5550,7 +5621,7 @@ mod tests { socket_bound: true, }, ) - .await; + .await; assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); assert_eq!(activation.admission.kind, Some(crate::api::model::ConnectionKind::Normal)); @@ -5608,10 +5679,7 @@ mod tests { // Simulate the counted lease being released before activation: // expire the session so it no longer has a counted lease. - app_state - .active_users - .terminate_session(&user.username, "tok-stale-followup") - .await; + app_state.active_users.terminate_session(&user.username, "tok-stale-followup").await; // Call activate with stale FollowUp. Must NOT skip admission — reclassification // to Activate must run so the placeholder is created. @@ -5631,7 +5699,7 @@ mod tests { socket_bound: true, }, ) - .await; + .await; // Must NOT skip — placeholder must be created since session is expired. assert!( @@ -5700,17 +5768,16 @@ mod tests { socket_bound: true, }, ) - .await; + .await; assert_eq!(activation.admission.permission, UserConnectionPermission::GracePeriod); assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-pre-resolved-grace") - .await; + let session = + app_state.active_users.get_and_update_user_session(&user.username, "tok-pre-resolved-grace").await; assert!( - session.is_some_and(|s| matches!(s.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })), + session + .is_some_and(|s| matches!(s.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })), "pre-resolved GracePeriod must materialize as PendingProvider lifecycle" ); } @@ -5756,7 +5823,7 @@ mod tests { socket_bound: true, }, ) - .await; + .await; // Prepare returns Allowed without running admission strategies. assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); @@ -5802,7 +5869,7 @@ mod tests { true, // prepare_only false, // terminate ) - .await; + .await; assert_eq!(request_class, PlaybackRequestClass::Prepare); // Prepare returns Allowed without running strategies. @@ -5852,10 +5919,7 @@ mod tests { .await; // Verify session exists. - let before = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let before = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; assert!(before.is_some(), "session should exist before terminate"); let (admission, grace_mode, request_class) = resolve_playback_request_admission( @@ -5870,17 +5934,14 @@ mod tests { false, // prepare_only true, // terminate ) - .await; + .await; assert_eq!(request_class, PlaybackRequestClass::Terminate); assert_eq!(admission.permission, UserConnectionPermission::Exhausted); assert_eq!(grace_mode, None); // Session should be expired after terminate. - let after = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await; + let after = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; assert!(after.is_none(), "session should be removed after terminate"); } @@ -5979,7 +6040,7 @@ mod tests { socket_bound: false, }, ) - .await; + .await; assert_eq!(activation.admission.permission, UserConnectionPermission::GracePeriod); assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); @@ -5989,15 +6050,16 @@ mod tests { .get_and_update_user_session(&user.username, "tok-grace-hold") .await .expect("placeholder session should exist"); - let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = - &session.lifecycle - else { + let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { panic!("grace hold should mark pending provider state") }; assert!(matches!(pending.reason_code, crate::api::model::PendingProviderReason::GraceHold)); assert!(pending.deadline >= pending.created_at); assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); - assert!(!session.lifecycle.is_counted(), "pending provider placeholder must not consume an active user lease before commit"); + assert!( + !session.lifecycle.is_counted(), + "pending provider placeholder must not consume an active user lease before commit" + ); } #[tokio::test] @@ -6059,7 +6121,7 @@ mod tests { socket_bound: false, }, ) - .await; + .await; assert_eq!(activation.admission.permission, UserConnectionPermission::Allowed); assert_eq!(activation.grace_mode, None); @@ -6087,10 +6149,7 @@ mod tests { enabled: true, name: "shared".to_string(), options: Some(ConfigTargetOptions { - share_live_streams: shared::model::ConfigTargetShareLiveStreams { - mpeg_ts: true, - ..Default::default() - }, + share_live_streams: shared::model::ConfigTargetShareLiveStreams { mpeg_ts: true, ..Default::default() }, ..ConfigTargetOptions::default() }), sort: None, @@ -6253,7 +6312,7 @@ mod tests { true, EvictionReentryGuard::Session("tok-new-request"), ) - .await; + .await; assert_eq!(result.admission.permission, UserConnectionPermission::GracePeriod, "grace should be granted"); assert!(matches!(result.grace_mode, Some(crate::api::model::GraceMode::Hold))); @@ -6268,10 +6327,7 @@ mod tests { // Strategies: [GraceHoldStream, EvictUserOldest] // Grace was used at index 0, so only EvictUserOldest (index 1) is evaluated. // Eviction frees the slot -> Allowed. - let strategies = vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - ]; + let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]; let grace_context = GraceResolutionContext { strategy_index: 0, strategies, kind: None }; let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { @@ -6286,10 +6342,7 @@ mod tests { throttle_str: None, throttle_kbps: 0, shared_burst_buffer_mb: 1, - admission_strategies: Some(vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - ]), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), }); let addr1: SocketAddr = "127.0.0.1:55701".parse().unwrap_or_else(|_| unreachable!()); @@ -6353,7 +6406,7 @@ mod tests { &grace_context, Some(crate::api::model::ConnectionKind::Normal), ) - .await; + .await; assert_eq!( result.admission.permission, @@ -6456,7 +6509,7 @@ mod tests { &grace_context, Some(crate::api::model::ConnectionKind::Normal), ) - .await; + .await; assert_eq!( result.admission.permission, @@ -6504,7 +6557,7 @@ mod tests { &grace_context, None, ) - .await; + .await; assert_eq!( result.admission.permission, @@ -6519,7 +6572,11 @@ mod tests { // Grace was at index 0, remaining slice is empty -> exhausted. // grace_context.kind is Soft — must be preserved in the exhausted result. let strategies = vec![AdmissionStrategy::GraceHoldStream]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies, kind: Some(crate::api::model::ConnectionKind::Soft) }; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies, + kind: Some(crate::api::model::ConnectionKind::Soft), + }; let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { retry: true, @@ -6553,7 +6610,7 @@ mod tests { &grace_context, Some(crate::api::model::ConnectionKind::Soft), ) - .await; + .await; assert_eq!( result.admission.permission, @@ -6657,7 +6714,7 @@ mod tests { &grace_context, Some(crate::api::model::ConnectionKind::Normal), ) - .await; + .await; assert_eq!( result.admission.permission, @@ -6672,7 +6729,11 @@ mod tests { // remaining slice is empty -> exhausted result must use original_kind. // This proves the empty-slice branch uses original_kind, not grace_context.kind. let strategies = vec![AdmissionStrategy::GraceHoldStream]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies, kind: Some(crate::api::model::ConnectionKind::Normal) }; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies, + kind: Some(crate::api::model::ConnectionKind::Normal), + }; let original_kind = Some(crate::api::model::ConnectionKind::Soft); let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { @@ -6707,12 +6768,11 @@ mod tests { &grace_context, original_kind, ) - .await; + .await; assert_eq!(result.admission.permission, UserConnectionPermission::Exhausted); assert_eq!( - result.admission.kind, - original_kind, + result.admission.kind, original_kind, "exhausted result must use original_kind (Soft), not grace_context.kind (Normal)" ); } @@ -6726,11 +6786,12 @@ mod tests { // When the helper returns Grace for the remaining strategy, the new // GraceResolutionContext.kind must be original_kind (Soft), not grace_context.kind (Normal). // This proves build_grace_ctx uses original_kind as source of truth. - let strategies = vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::GraceInstantStream, - ]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies, kind: Some(crate::api::model::ConnectionKind::Normal) }; + let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies, + kind: Some(crate::api::model::ConnectionKind::Normal), + }; let original_kind = Some(crate::api::model::ConnectionKind::Soft); let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { @@ -6745,10 +6806,7 @@ mod tests { throttle_str: None, throttle_kbps: 0, shared_burst_buffer_mb: 1, - admission_strategies: Some(vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::GraceInstantStream, - ]), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]), }); let addr1: SocketAddr = "127.0.0.1:55710".parse().unwrap_or_else(|_| unreachable!()); @@ -6812,17 +6870,14 @@ mod tests { &grace_context, original_kind, ) - .await; + .await; assert_eq!( result.admission.permission, UserConnectionPermission::GracePeriod, "remaining GraceInstantStream should grant GracePeriod" ); - assert!( - result.grace_context.is_some(), - "grace_context must be present when grace is granted" - ); + assert!(result.grace_context.is_some(), "grace_context must be present when grace is granted"); assert_eq!( result.grace_context.as_ref().unwrap().kind, original_kind, @@ -6845,10 +6900,7 @@ mod tests { throttle_str: None, throttle_kbps: 0, shared_burst_buffer_mb: 1, - admission_strategies: Some(vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - ]), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), }); let first_addr: std::net::SocketAddr = "127.0.0.1:55151".parse().unwrap_or_else(|_| unreachable!()); @@ -6944,7 +6996,7 @@ mod tests { false, EvictionReentryGuard::Session("tok-third"), ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; @@ -7007,7 +7059,7 @@ mod tests { false, EvictionReentryGuard::Session("vod-session"), ) - .await; + .await; assert_eq!(session_based.admission.permission, UserConnectionPermission::Allowed); let connection_based = resolve_admission_with_strategies( @@ -7022,7 +7074,7 @@ mod tests { false, EvictionReentryGuard::Session("vod-session"), ) - .await; + .await; assert_eq!(connection_based.admission.permission, UserConnectionPermission::Exhausted); } @@ -7110,7 +7162,7 @@ mod tests { false, EvictionReentryGuard::SocketPlayback { virtual_id: 9001 }, ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; @@ -7238,7 +7290,7 @@ mod tests { false, EvictionReentryGuard::SocketPlayback { virtual_id: 9103 }, ) - .await; + .await; let admission = result.admission; assert_eq!(admission.permission, UserConnectionPermission::Allowed); @@ -7361,7 +7413,7 @@ mod tests { false, EvictionReentryGuard::Session("session-other"), ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; @@ -7454,7 +7506,7 @@ mod tests { false, EvictionReentryGuard::SocketPlayback { virtual_id: 9201 }, ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; @@ -7510,8 +7562,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let active_streams = app_state.active_users.active_streams().await; assert_eq!(active_streams.len(), 1, "local file streaming should register an active stream"); @@ -7570,8 +7622,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let _second_response = local_stream_response( &second_fingerprint, @@ -7587,8 +7639,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); assert_eq!(app_state.active_users.active_streams().await.len(), 1); @@ -7634,7 +7686,7 @@ mod tests { 0, crate::api::model::ConnectionKind::Normal, ) - .await; + .await; assert!(registered.is_some(), "shared stream should register"); let mut user = ProxyUserCredentials::default(); @@ -7692,8 +7744,8 @@ mod tests { false, None, ) - .await - .into_response(); + .await + .into_response(); assert_eq!(response.status(), StatusCode::OK); let session_admission = app_state @@ -7720,10 +7772,7 @@ mod tests { let addr = "127.0.0.1:55143".parse().unwrap_or_else(|_| unreachable!()); let fingerprint = create_test_fingerprint(addr); let input_name = "provider_1".intern(); - let input = app_state - .app_config - .get_input_by_name(&input_name) - .expect("provider input should exist"); + let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); let target = Arc::new(ConfigTarget { id: 1, enabled: true, @@ -7763,8 +7812,8 @@ mod tests { false, None, ) - .await - .into_response(); + .await + .into_response(); // Custom-video stream is enabled (`custom_stream_response_enabled: true` // in this fixture), so a missing resource must return 400 — the @@ -7777,11 +7826,7 @@ mod tests { "failed provider open must rollback provisional user activation" ); assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, "rollback-session") - .await - .is_none(), + app_state.active_users.get_and_update_user_session(&user.username, "rollback-session").await.is_none(), "failed provider open must remove the provisional placeholder session" ); } @@ -7808,6 +7853,7 @@ mod tests { disable_provider_grace: false, reconnect_flag: None, provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, }; assert!( @@ -7817,12 +7863,7 @@ mod tests { let provisioning_details = StreamDetails { stream: None, - stream_info: Some(( - Vec::new(), - StatusCode::OK, - None, - Some(CustomVideoStreamType::Provisioning), - )), + stream_info: Some((Vec::new(), StatusCode::OK, None, Some(CustomVideoStreamType::Provisioning))), provider_name: Some("provider_1".intern()), request_url: None, session_headers: None, @@ -7831,6 +7872,7 @@ mod tests { disable_provider_grace: false, reconnect_flag: None, provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, }; assert!( @@ -7856,6 +7898,7 @@ mod tests { disable_provider_grace: false, reconnect_flag: None, provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, }; assert!( @@ -7908,8 +7951,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); assert!(!should_compress_response(&response)); } @@ -7959,8 +8002,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let _second_response = local_stream_response( &second_fingerprint, @@ -7976,8 +8019,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let active_streams = app_state.active_users.active_streams().await; assert_eq!(active_streams.len(), 1, "stable playback token should reuse the tracked local connection"); @@ -8032,8 +8075,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let second_response = local_stream_response( &second_fingerprint, @@ -8049,8 +8092,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); assert_eq!(second_response.status(), StatusCode::OK); @@ -8108,8 +8151,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); let second_response = local_stream_response( &second_fingerprint, @@ -8125,8 +8168,8 @@ mod tests { None, false, ) - .await - .into_response(); + .await + .into_response(); assert_eq!(second_response.status(), StatusCode::OK); @@ -8195,7 +8238,7 @@ mod tests { true, EvictionReentryGuard::Session("tok-hls-first"), ) - .await; + .await; let second_admission = resolve_admission_with_strategies( &app_state, &user.username, @@ -8208,7 +8251,7 @@ mod tests { true, EvictionReentryGuard::Session("tok-hls-second"), ) - .await; + .await; assert_eq!(first_admission.admission.permission, UserConnectionPermission::Allowed); assert_eq!(second_admission.admission.permission, UserConnectionPermission::Allowed); @@ -8364,7 +8407,7 @@ mod tests { false, EvictionReentryGuard::SocketPlayback { virtual_id: 5001 }, ) - .await; + .await; let admission = result.admission; let grace_mode = result.grace_mode; @@ -8382,11 +8425,7 @@ mod tests { )) ); assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, "tok-hls-preserved") - .await - .is_none(), + app_state.active_users.get_and_update_user_session(&user.username, "tok-hls-preserved").await.is_none(), "preserved session should be removed once the TS request evicts it" ); } @@ -8530,8 +8569,10 @@ mod tests { let first = Fingerprint::new("10.0.0.6|player".to_string(), "10.0.0.6".to_string(), first_addr); let second = Fingerprint::new(first.key.clone(), first.client_ip.clone(), second_addr); - let first_token = create_playback_session_fingerprint(&first, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); - let second_token = create_playback_session_fingerprint(&second, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); + let first_token = + create_playback_session_fingerprint(&first, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); + let second_token = + create_playback_session_fingerprint(&second, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); assert_eq!(first_token, second_token); assert!(first_token.contains(&first.key)); @@ -8651,11 +8692,7 @@ mod tests { app_state.connection_manager.release_connection(&hls_addr).await; assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, &hls_token) - .await - .is_some(), + app_state.active_users.get_and_update_user_session(&user.username, &hls_token).await.is_some(), "preserved HLS session should still exist before the competing TS request" ); assert_eq!( @@ -8668,11 +8705,7 @@ mod tests { "the preserved HLS playback must still reserve the user's only slot before the TS request is evaluated" ); assert_eq!( - app_state - .active_users - .get_eviction_candidates(&user.username, &ts_fingerprint.client_ip) - .await - .len(), + app_state.active_users.get_eviction_candidates(&user.username, &ts_fingerprint.client_ip).await.len(), 1, "the preserved HLS playback should be the single eviction candidate for the competing TS request" ); @@ -8689,7 +8722,7 @@ mod tests { false, false, ) - .await; + .await; assert_eq!(request_class, PlaybackRequestClass::Activate); assert_eq!(ts_admission.permission, UserConnectionPermission::Allowed); assert_eq!(ts_grace_mode, None); @@ -8744,11 +8777,7 @@ mod tests { "after the competing TS request, the old Xtream HLS session must be gone so later /hls segment fetches cannot revive it" ); assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, &ts_token) - .await - .is_some(), + app_state.active_users.get_and_update_user_session(&user.username, &ts_token).await.is_some(), "the winning TS playback should remain tracked under its socket-bound Xtream token" ); } @@ -8789,6 +8818,66 @@ mod tests { assert_eq!(session_reacquire_cleanup_addrs(&session, &seek), vec![primary, overlap]); } + #[tokio::test] + async fn intentional_deferred_open_retains_provider_grace_handle() { + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .await + .expect("holder occupies the provider slot"); + let input = app_state.app_config.get_input_by_name(&provider_name).expect("provider input"); + let stream_url = "http://provider-1.example/live/user1/pass1/100.m3u8"; + let mut channel = create_test_live_channel(stream_url); + channel.item_type = PlaylistItemType::LiveHls; + let fingerprint = create_test_fingerprint(deferred_addr); + + let mut details = create_stream_response_details( + &app_state, + &get_stream_options(&app_state), + stream_url, + "deferred-user", + &fingerprint, + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::LiveHls, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + false, + UserConnectionPermission::Allowed, + None, + true, + true, + channel.virtual_id, + 0, + crate::api::model::ConnectionKind::Normal, + false, + Some("deferred-session"), + None, + false, + Some(true), + None, + ) + .await + .expect("provider grace creates deferred stream details"); + + assert!(details.stream.is_none()); + assert!(details.has_deferred_provider_open()); + assert!(details.provider_handle.is_some(), "deferred open must retain its provider allocation"); + + app_state.connection_manager.release_provider_handle(details.provider_handle.take()).await; + app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; + } + #[test] fn grace_hold_defers_live_but_not_provider_affine_session_reopens() { assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::LiveHls, false)); @@ -8806,10 +8895,8 @@ mod tests { cleanup_forced_reopen_addrs(&app_state, PlaylistItemType::LiveHls, &[addr]).await; - let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()) - .await - .ok() - .and_then(Result::ok); + let signal = + tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); assert!(signal.is_none(), "adaptive cleanup should not hard-close the previous client socket"); } @@ -8821,10 +8908,8 @@ mod tests { cleanup_forced_reopen_addrs(&app_state, PlaylistItemType::Live, &[addr]).await; - let signal = tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()) - .await - .ok() - .and_then(Result::ok); + let signal = + tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); assert!(matches!( signal, Some(crate::api::model::CloseConnectionSignal::WithReason(signal_addr, _)) if signal_addr == addr @@ -8870,6 +8955,7 @@ mod tests { input_name: "provider_with_flag".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "1".intern(), }; let hls_ext = shared::defaults::HLS_EXT.to_string(); @@ -8906,9 +8992,7 @@ mod tests { } /// `Arc>` with no `GeoIP` database loaded. - fn empty_geoip() -> Arc> { - Arc::new(ArcSwapOption::::default()) - } + fn empty_geoip() -> Arc> { Arc::new(ArcSwapOption::::default()) } /// `Arc>` with a mock `GeoIP` that always reports the /// given country for any lookup. @@ -8985,10 +9069,7 @@ mod tests { #[test] fn country_match_allows() { assert_network_decision( - Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - }), + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), &mock_geoip("DE"), "8.8.8.8", NetworkAccessDecision::Allowed, @@ -8998,10 +9079,7 @@ mod tests { #[test] fn country_miss_denies() { assert_network_decision( - Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - }), + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), &mock_geoip("US"), "8.8.8.8", NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch), @@ -9011,10 +9089,7 @@ mod tests { #[test] fn no_geoip_denies_on_country_restriction() { assert_network_decision( - Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - }), + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), &empty_geoip(), "8.8.8.8", NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable), @@ -9024,10 +9099,7 @@ mod tests { #[test] fn ipv4_vs_ipv6_denies_gracefully() { assert_network_decision( - Some(NetworkAccess { - allowed_countries: vec![], - allowed_networks: vec!["2001:db8::/32".parse().unwrap()], - }), + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["2001:db8::/32".parse().unwrap()] }), &empty_geoip(), "192.168.1.1", NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), @@ -9116,14 +9188,20 @@ mod tests { allowed_networks: vec![], })); let mock_geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new("DE"))))); - assert_eq!(evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Deny), NetworkAccessDecision::Allowed); + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Allowed + ); } #[test] fn network_denied_reason_none_when_no_config() { let user = user_with_network_access(None); let geoip = Arc::new(ArcSwapOption::::default()); - assert_eq!(evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), NetworkAccessDecision::Allowed); + assert_eq!( + evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Allowed + ); } // ========================================================================================= @@ -9229,6 +9307,9 @@ mod tests { NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable) ); // Allow policy should return AllowedGeoIpUnavailable - assert_eq!(evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow), NetworkAccessDecision::AllowedGeoIpUnavailable); + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow), + NetworkAccessDecision::AllowedGeoIpUnavailable + ); } } diff --git a/backend/src/api/endpoints/custom_video_stream_api.rs b/backend/src/api/endpoints/custom_video_stream_api.rs index e8d57fb76..6dd2e2152 100644 --- a/backend/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/src/api/endpoints/custom_video_stream_api.rs @@ -3,7 +3,7 @@ use crate::{ api_utils::{create_api_proxy_user, mark_response_as_uncompressed}, endpoints::hls_api::{ build_virtual_hls_entry_path, hls_panel_provisioning_poll_manifest_response, - resolve_hls_virtual_input_for_target, + resolve_hls_virtual_source_for_target, }, model::{ create_custom_video_stream_response, hls_custom_video_manifest_response_with_virtual_id, @@ -410,19 +410,20 @@ async fn cvs_provisioning_manifest_api( Err(response) => return *response, }; - let Some(input) = resolve_hls_virtual_input_for_target(&app_state, &target, query.id).await else { - return StatusCode::NOT_FOUND.into_response(); + let source = match resolve_hls_virtual_source_for_target(&app_state, &target, query.id).await { + Ok(source) => source, + Err(status) => return status.into_response(), }; - let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, query.id); + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &source.input, &user, query.id); let server_path = app_state.app_config.get_user_server_info(&user).and_then(|server| server.path); hls_panel_provisioning_poll_manifest_response( &app_state, &fingerprint, &user, &target, - &input, - query.id, + &source.input, + &source.stream_identity, &original_hls_entry_path, server_path.as_deref(), ) diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index 6f1e8c508..f28815c4b 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -16,49 +16,44 @@ use crate::{ }, model::{ begin_hls_origin_account_io_bounded, build_hls_origin_session_owner, build_proxy_session_id, - cold_start_retry_after_seconds, hls_cached_manifest_options_for_requirement, - extract_hls_provider_session_headers, hls_committed_manifest_body_for_request, - hls_custom_video_manifest_path, - hls_custom_video_manifest_response_with_virtual_id, hls_object_body_deadline, hls_origin_account_status, - hls_provisioning_discontinuity_sequence, is_custom_video_stream_enabled, - is_hls_provisioning_gap_segment, is_hls_provisioning_segment, - hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, - hls_transient_object_fetch_failure, hls_transient_origin_response, - hls_virtual_entry_redirect_response, maybe_trigger_origin_refresh, new_hls_access_lease_id, - origin_account_binding_from_allocation, retry_after_secs_from_ms, safe_hls_access_lease_id, + cold_start_retry_after_seconds, extract_hls_provider_session_headers, + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, force_identity_without_range, + hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request, + hls_custom_video_manifest_path, hls_custom_video_manifest_response_with_virtual_id, + hls_manifest_commit_requirement, hls_object_body_deadline, hls_origin_account_status, + hls_provisioning_discontinuity_sequence, hls_should_wait_for_initial_manifest_commit, + hls_transient_object_fetch_failure, hls_transient_origin_response, hls_virtual_entry_redirect_response, + is_custom_video_stream_enabled, is_hls_provisioning_gap_segment, is_hls_provisioning_segment, + maybe_trigger_origin_refresh, new_hls_access_lease_id, origin_account_binding_from_allocation, + record_successful_transient_segment_fetch, record_temporary_transient_segment_fetch_failure, + resolve_hls_transient_object_cache_action, retry_after_secs_from_ms, safe_hls_access_lease_id, safe_proxy_session_id, safe_user_session_token, scrub_hls_origin_headers, serve_hls_map_cache_outcome, serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, serve_hls_transient_object_cache_response, should_remove_hls_origin_header, - start_hls_panel_provisioning_once, - try_hls_panel_provisioning_manifest_response, validate_hls_access_lease, AppState, CacheAccessState, - CustomVideoStreamType, HlsAccessAdmissionMode, + start_hls_panel_provisioning_once, try_hls_panel_provisioning_manifest_response, validate_hls_access_lease, + AppState, CacheAccessState, ConnectionHistoryMode, CustomVideoStreamType, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAccountBindingProtection, - HlsCacheResponseContext, HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, HlsCachedManifestOptions, - HlsCommittedManifestBody, HlsEffectiveOriginAcquirePolicy, HlsManifestCommitRequirement, HlsMapFile, - HlsMediaActivityMarker, - HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, - HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginResourceClients, - HlsOriginResourceFetchError, HlsOriginSource, - HlsOriginSourceKind, HlsOriginWorkClass, HlsPanelProvisioningRedirectPaths, HlsPlaybackFamilyKey, - HlsProvisioningStatus, HlsQosMeterInit, HlsQosRuntimeConfig, - HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, - HLS_PROVISIONING_TARGET_DURATION_SECS, - HlsResourceServeFailure, HlsResourceServeOutcome, HlsSegmentFailureObject, - HlsSegmentFailureTransition, HlsSegmentFile, - HlsSession, HlsSessionHandle, HlsSessionKey, - HlsSessionMode, HlsSessionStoreOutcome, LiveHlsOriginEntry, OriginRefreshRequest, OriginSegmentKey, - ProviderAllocation, + HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, HlsCacheResponseContext, + HlsCachedManifestOptions, HlsCommittedManifestBody, HlsEffectiveOriginAcquirePolicy, + HlsManifestCommitRequirement, HlsMapFile, HlsMediaActivityMarker, HlsOriginAccountBinding, + HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, + HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginSource, HlsOriginSourceKind, + HlsOriginWorkClass, HlsPanelProvisioningRedirectPaths, HlsPlaybackFamilyKey, HlsProvisioningStatus, + HlsQosMeterInit, HlsQosRuntimeConfig, HlsResourceServeFailure, HlsResourceServeOutcome, HlsSegmentFile, + HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, + HlsTransientCacheCommitContext, HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, + HlsTransientObjectCacheAction, HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, + HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, + LiveHlsOriginEntry, OriginRefreshRequest, OriginSegmentKey, ProviderAllocation, ProviderConfig as RuntimeProviderConfig, ProviderHandle, ProxySessionId, RetryPolicy, SegmentCacheKey, - SegmentCacheStatus, SegmentDemandFetchOutcome, SegmentFetchContext, SegmentFetchPolicy, TransientObjectCacheKey, - TransientObjectUnavailableState, TransientResourceFile, TransientResourceRef, - TransientResourceKind, TransportStreamBuffer, UserSession, ConnectionHistoryMode, StreamMeterHandle, - HLS_ACCESS_LEASE_ID_PLACEHOLDER, HlsTransientObjectFetchFailure, HlsTransientOriginIoGuard, - HlsTransientCacheCommitContext, HlsTransientObjectCacheAction, HlsTransientObjectFetchFinalizer, - HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, - fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, - fetch_hls_transient_origin_response_with_attempt_prepare, resolve_hls_transient_object_cache_action, + SegmentCacheStatus, SegmentDemandFetchOutcome, SegmentFetchContext, SegmentFetchPolicy, StreamMeterHandle, + TransientObjectCacheKey, TransientObjectUnavailableState, TransientResourceFile, TransientResourceRef, + TransportStreamBuffer, UserSession, HLS_ACCESS_LEASE_ID_PLACEHOLDER, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, + HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, + MAX_HLS_MANIFEST_BYTES, }, panel_api::can_provision_on_exhausted, }, @@ -68,12 +63,12 @@ use crate::{ ReverseProxyDisabledHeaderConfig, }, processing::parser::hls::{ - get_hls_session_token_and_url_from_token, rewrite_hls, + get_hls_session_token_and_url_from_token, initial_strip::{materialize_initial_hls_strip_view, HlsInitialStripOutcome}, - RewriteHlsProps, + rewrite_hls, RewriteHlsProps, }, repository::{m3u_get_item_for_stream_id, storage_const, xtream_get_item_for_stream_id}, - utils::{debug_if_enabled, request, request::is_file_url}, + utils::{content_coding::OutboundContentCodingPolicy, debug_if_enabled, request, request::is_file_url}, }; use axum::{ body::Body, @@ -85,15 +80,15 @@ use log::{debug, error, warn}; use serde::Deserialize; use sha2::{Digest, Sha256}; use shared::{ + defaults::HLS_EXT, model::{ - ConnectFailureReason, FailureStage, InputType, PlaylistItemType, StreamChannel, StreamInfo, TargetType, - UserConnectionPermission, XtreamCluster, + ConnectFailureReason, FailureStage, InputType, PlaylistEntry, PlaylistItemType, StreamChannel, StreamInfo, + TargetType, UserConnectionPermission, XtreamCluster, }, utils::{ generate_random_string, is_hls_url, replace_url_extension, sanitize_sensitive_info, Internable, PROVIDER_SCHEME_PREFIX, }, - defaults::{HLS_EXT,}, }; use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; use url::Url; @@ -281,7 +276,11 @@ fn split_hls_line_ending(part: &str) -> (&str, &str) { } } -fn materialize_hls_access_manifest(hls_content: &str, lease_id: &HlsAccessLeaseId, server_path: Option<&str>) -> String { +fn materialize_hls_access_manifest( + hls_content: &str, + lease_id: &HlsAccessLeaseId, + server_path: Option<&str>, +) -> String { let hls_content = hls_content.replace(HLS_ACCESS_LEASE_ID_PLACEHOLDER, &lease_id.0); apply_hls_proxy_public_path_prefix(hls_content, server_path) } @@ -342,9 +341,7 @@ fn materialize_shared_hls_access_manifest( fn hls_access_lease_ttl_ms(app_state: &Arc) -> u64 { app_state.hls_proxy.session_idle_timeout_ms() } -fn duration_to_millis_saturating(duration: Duration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} +fn duration_to_millis_saturating(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } fn hls_pending_bootstrap_window_ms() -> u64 { duration_to_millis_saturating(hls_initial_manifest_decision_wait_timeout()) @@ -375,12 +372,7 @@ async fn touch_pending_manifest_follow_up_window( let now_ms = current_time_millis(); let _ = app_state .hls_proxy - .mark_pending_manifest_follow_up_for_lease( - access_lease_id, - &proxy_session_id, - now_ms, - target_duration, - ) + .mark_pending_manifest_follow_up_for_lease(access_lease_id, &proxy_session_id, now_ms, target_duration) .await; } @@ -472,11 +464,7 @@ async fn hls_cache_response_context( access_context: &HlsAccessContext, now_ms: u64, ) -> HlsCacheResponseContext { - let qos_meter = app_state - .hls_proxy - .qos() - .meter_for_access_lease(&access_context.lease_id) - .await; + let qos_meter = app_state.hls_proxy.qos().meter_for_access_lease(&access_context.lease_id).await; HlsCacheResponseContext::new( access_context.lease_id.clone(), app_state.hls_proxy.cache_duration_seconds(), @@ -906,9 +894,8 @@ async fn fetch_or_passthrough_transient_resource( }) .await; match fetch_result.result { - Ok((response, transient_origin_guard)) => { - let media_activity_marker = if response.status().is_success() { - record_successful_transient_segment_fetch(session, resource).await; + Ok(response) => { + let media_activity_marker = if response.decoded.status.is_success() { mark_hls_authorized_media_access(app_state, session, now_ms).await; let _ = ensure_hls_cache_stream_registered(app_state, fingerprint, headers, access_context, session).await; @@ -918,15 +905,15 @@ async fn fetch_or_passthrough_transient_resource( }; hls_transient_origin_response( response, - Arc::clone(&resource.access), - transient_origin_guard, - media_activity_marker, - now_ms, - proxy_session_id.0.clone(), - resource_file.resource_id.0.clone(), - resource.kind, - resource.resolved_origin_uri.clone(), - app_state.hls_proxy.segment_fetch_policy().origin_segment_timeout_ms, + HlsTransientDirectResponseContext { + hls_proxy: Arc::clone(&app_state.hls_proxy), + session: Arc::clone(session), + resource: resource.clone(), + policy: policy.clone(), + media_activity_marker, + now_ms, + proxy_session_id: proxy_session_id.clone(), + }, ) } Err(err) => { @@ -982,7 +969,7 @@ struct HlsTransientEndpointOriginFetchRequest<'a> { } struct HlsTransientOriginFetchResult { - result: Result<(reqwest::Response, Option), HlsOriginResourceFetchError>, + result: Result>, HlsOriginResourceFetchError>, runtime_prepare_error: Option, } @@ -1013,38 +1000,33 @@ async fn fetch_transient_origin_response_with_provider_io( let fingerprint_for_prepare = request.fingerprint.clone(); let headers_for_prepare = request.headers.clone(); let runtime_prepare_error_for_prepare = Arc::clone(&runtime_prepare_error); - let result = fetch_hls_transient_origin_response_with_attempt_prepare( - fetch_request, - move |_attempt| { - let app_state = Arc::clone(&app_state_for_prepare); - let session = Arc::clone(&session_for_prepare); - let access_context = access_context_for_prepare.clone(); - let fingerprint = fingerprint_for_prepare.clone(); - let headers = headers_for_prepare.clone(); - let runtime_prepare_error = Arc::clone(&runtime_prepare_error_for_prepare); - async move { - match prepare_hls_transient_origin_io_for_authorized_resource_work( - &app_state, - &session, - &access_context, - &fingerprint, - &headers, - current_time_millis(), - ) - .await - { - Ok(guard) => Ok(guard), - Err(err) => { - *runtime_prepare_error.lock().await = Some(err); - Err(HlsOriginResourceFetchError::ProviderUnavailable( - HlsBoundAccountAcquireErrorKind::Unavailable, - )) - } + let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, move |_attempt| { + let app_state = Arc::clone(&app_state_for_prepare); + let session = Arc::clone(&session_for_prepare); + let access_context = access_context_for_prepare.clone(); + let fingerprint = fingerprint_for_prepare.clone(); + let headers = headers_for_prepare.clone(); + let runtime_prepare_error = Arc::clone(&runtime_prepare_error_for_prepare); + async move { + match prepare_hls_transient_origin_io_for_authorized_resource_work( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + current_time_millis(), + ) + .await + { + Ok(guard) => Ok(guard), + Err(err) => { + *runtime_prepare_error.lock().await = Some(err); + Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable)) } } - .boxed() - }, - ) + } + .boxed() + }) .await; let runtime_prepare_error = *runtime_prepare_error.lock().await; HlsTransientOriginFetchResult { result, runtime_prepare_error } @@ -1173,10 +1155,7 @@ fn safe_transient_resource_id(resource_id: &crate::api::model::TransientResource // Truncate at the first char boundary at or before byte 8 to avoid allocating // a temporary `String` of 8 chars (and a second UTF-8 walk via `len()`). let full = resource_id.0.as_str(); - let truncate_at = full - .char_indices() - .nth(8) - .map_or(full.len(), |(byte_idx, _)| byte_idx); + let truncate_at = full.char_indices().nth(8).map_or(full.len(), |(byte_idx, _)| byte_idx); if truncate_at == full.len() { return full.to_owned(); } @@ -1294,20 +1273,23 @@ async fn ensure_hls_cache_stream_registered( app_state .connection_manager - .update_connection_with_history_mode(crate::api::model::ConnectionParams { - meter_uid: qos_registration.meter_uid, - username: &access.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind, - priority, - soft_priority: user.soft_priority, - fingerprint, - provider, - stream_channel: &stream_channel, - user_agent, - session_token: Some(&access.user_session_token), - }, history_mode) + .update_connection_with_history_mode( + crate::api::model::ConnectionParams { + meter_uid: qos_registration.meter_uid, + username: &access.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority, + soft_priority: user.soft_priority, + fingerprint, + provider, + stream_channel: &stream_channel, + user_agent, + session_token: Some(&access.user_session_token), + }, + history_mode, + ) .await } @@ -1470,12 +1452,14 @@ fn hls_access_lease_response_flag_manifest_response( flag: &HlsAccessLeaseResponseFlag, ) -> axum::response::Response { match flag { - HlsAccessLeaseResponseFlag::ChannelUnavailable { .. } => hls_custom_video_manifest_redirect_response_for_username( - app_state, - username, - CustomVideoStreamType::ChannelUnavailable, - StatusCode::NOT_FOUND, - ), + HlsAccessLeaseResponseFlag::ChannelUnavailable { .. } => { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) + } } } @@ -1589,8 +1573,7 @@ async fn hls_manifest_access_lease_validation_response( } else { None }; - let username = - lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); + let username = lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); match err { HlsAccessLeaseValidationError::AdmissionDenied => username.map_or_else( || StatusCode::FORBIDDEN.into_response(), @@ -1625,8 +1608,7 @@ async fn hls_resource_access_lease_validation_response( } else { None }; - let username = - lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); + let username = lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); match err { HlsAccessLeaseValidationError::AdmissionDenied => username.map_or_else( || StatusCode::FORBIDDEN.into_response(), @@ -1681,14 +1663,16 @@ async fn hls_manifest_access_context_and_state( safe_hls_access_lease_id(access_lease_id), safe_proxy_session_id(proxy_session_id) ); - return Err(Box::new(hls_manifest_access_lease_validation_response( - app_state, - proxy_session_id, - access_lease_snapshot, - now_ms, - err, - ) - .await)); + return Err(Box::new( + hls_manifest_access_lease_validation_response( + app_state, + proxy_session_id, + access_lease_snapshot, + now_ms, + err, + ) + .await, + )); } }; if access_lease_snapshot.is_none() @@ -1778,63 +1762,6 @@ async fn hls_transient_object_unavailable_response( } } -async fn record_successful_transient_segment_fetch(session: &HlsSessionHandle, resource: &crate::api::model::TransientResourceRef) { - if resource.kind != TransientResourceKind::Segment { - return; - } - let mut session = session.write().await; - if let Some(reset_failures) = session.record_successful_segment_fetch() { - debug!( - "HLS segment temporary failure counter reset: session={} previous_failures={reset_failures}", - safe_proxy_session_id(&session.proxy_session_id) - ); - } -} - -async fn record_temporary_transient_segment_fetch_failure( - session: &HlsSessionHandle, - resource: &crate::api::model::TransientResourceRef, - policy: &SegmentFetchPolicy, - now_ms: u64, -) -> Option { - if resource.kind != TransientResourceKind::Segment { - return None; - } - let mut session = session.write().await; - let threshold = session.segment_temporary_failure_threshold(policy.permanent_failure_segment_threshold); - match session.record_temporary_segment_fetch_failure( - now_ms, - HlsSegmentFailureObject::Transient { - resource_id: resource.id.0.clone(), - }, - threshold, - ) { - HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { - debug!( - "HLS segment temporary failure counted: session={} object={} failures={} threshold={}", - safe_proxy_session_id(&session.proxy_session_id), - resource.id.0, - failures, - threshold - ); - None - } - HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { - warn!( - "HLS segment temporary failure threshold reached: session={} failures={} threshold={}", - safe_proxy_session_id(&session.proxy_session_id), - failures, - threshold - ); - session.invalidate_queued_origin_work(); - Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectTemporaryFailureThreshold { - failures, - threshold, - }) - } - } -} - #[allow(clippy::too_many_lines)] async fn fetch_and_cache_transient_origin_response( context: HlsTransientEndpointCacheFetchContext<'_>, @@ -1855,7 +1782,7 @@ async fn fetch_and_cache_transient_origin_response( resolved_origin_uri: context.resource.resolved_origin_uri.clone(), origin_headers: context.origin_headers.clone(), origin_provider_session_headers: context.origin_provider_session_headers.clone(), - range_header: context.range_header.clone(), + range_header: None, resource_file: context.resource_file.clone(), resource_kind: context.resource.kind, clients, @@ -1872,9 +1799,7 @@ async fn fetch_and_cache_transient_origin_response( resource: context.resource.clone(), resource_file: context.resource_file.clone(), cache_key: context.cache_key.clone(), - range_header: context.range_header.clone(), cache_duration_ms: context.cache_duration_ms, - origin_segment_timeout_ms: policy.origin_segment_timeout_ms, }, }; let app_state_for_prepare = Arc::clone(context.app_state); @@ -1970,9 +1895,13 @@ async fn fetch_and_cache_transient_origin_response( let mut response_flag_reason = None; match final_failure { HlsTransientObjectFetchFailure::Retryable => { - if let Some(reason) = - record_temporary_transient_segment_fetch_failure(context.session, context.resource, &policy, failed_at_ms) - .await + if let Some(reason) = record_temporary_transient_segment_fetch_failure( + context.session, + context.resource, + &policy, + failed_at_ms, + ) + .await { response_flag_reason = Some(reason); context.session.write().await.transient.mark_object_failed_permanent( @@ -1994,9 +1923,8 @@ async fn fetch_and_cache_transient_origin_response( failed_at_ms, status, ); - response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectPermanentFailure { - status, - }); + response_flag_reason = + Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectPermanentFailure { status }); } } if let Some(reason) = response_flag_reason { @@ -2100,9 +2028,45 @@ fn build_hls_manifest_request_headers( let mut headers = request::get_request_headers(Some(&input_headers), Some(&forwarded), disabled_headers, default_user_agent); scrub_hls_origin_headers(&mut headers, disabled_headers); + force_identity_without_range(&mut headers); headers } +async fn download_legacy_hls_manifest( + app_state: &Arc, + input: &InputSource, + headers: &HeaderMap, +) -> Result<(String, String, HeaderMap), std::io::Error> { + let deadline = Duration::from_millis(app_state.hls_proxy.origin_manifest_timeout_ms().max(1)); + let fetch_options = request::RequestFetchOptions::with_attempt_idle_timeout(deadline) + .with_content_coding(OutboundContentCodingPolicy::Identity); + let body_options = request::TextContentBodyOptions::hls_manifest(MAX_HLS_MANIFEST_BYTES, deadline); + let options = request::TextContentFetchOptions::new(fetch_options, body_options); + + if app_state.should_use_manual_redirects() { + request::download_text_content_with_manual_redirects_and_headers_and_options( + &app_state.app_config, + &app_state.http_client_no_redirect.load(), + input, + Some(headers), + false, + MAX_MANUAL_REDIRECTS, + options, + ) + .await + } else { + request::download_text_content_with_headers_and_options( + &app_state.app_config, + &app_state.http_client.load(), + input, + Some(headers), + false, + options, + ) + .await + } +} + struct HlsCacheManifestOrigin<'a> { raw_request_url: &'a str, session_entry_url: HlsOriginEntryUrl, @@ -2177,12 +2141,42 @@ fn is_supported_hls_origin_url(input: &ConfigInput, url: &str) -> bool { input.get_resolve_provider(url).is_some() || is_http_hls_origin_url(url) } -fn hls_stream_ref_from_virtual_id(virtual_id: u32) -> String { virtual_id.to_string() } - fn build_hls_origin_source(input: &ConfigInput, stream_ref: impl Into) -> HlsOriginSource { HlsOriginSource::new(input.id, Arc::clone(&input.name), stream_ref, hls_origin_source_kind(input.input_type)) } +/// Keeps target routing identity separate from the immutable input content identity. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(in crate::api) struct HlsEntryStreamIdentity { + virtual_id: u32, + input_stream_id: Arc, +} + +impl HlsEntryStreamIdentity { + pub(in crate::api) fn new(virtual_id: u32, input_stream_id: impl Into>) -> Option { + let input_stream_id = input_stream_id.into(); + if input_stream_id.trim().is_empty() { + return None; + } + Some(Self { virtual_id, input_stream_id }) + } + + pub(in crate::api) fn from_playlist_item(item: &impl PlaylistEntry) -> Option { + Self::new(item.get_virtual_id(), item.get_input_stream_id()?) + } + + pub(in crate::api) const fn virtual_id(&self) -> u32 { self.virtual_id } + + fn stream_ref(&self) -> &str { self.input_stream_id.as_ref() } +} + +/// Resolves the configured input together with both identities of one target entry. +#[derive(Debug, Clone)] +pub(in crate::api) struct HlsResolvedVirtualSource { + pub(in crate::api) input: Arc, + pub(in crate::api) stream_identity: HlsEntryStreamIdentity, +} + fn hls_origin_source_kind(input_type: InputType) -> HlsOriginSourceKind { if input_type.is_xtream() { HlsOriginSourceKind::XtreamLive @@ -2331,9 +2325,7 @@ fn rewrite_hls_path_auth_fields( new_username: &str, new_password: &str, ) -> bool { - let Some(mut segments) = url - .path_segments() - .map(|segments| segments.map(ToOwned::to_owned).collect::>()) + let Some(mut segments) = url.path_segments().map(|segments| segments.map(ToOwned::to_owned).collect::>()) else { return false; }; @@ -2419,9 +2411,7 @@ impl HlsOriginRuntimeNoAccountReason { } } -fn hls_no_account_reason_for_binding( - binding: Option<&HlsOriginAccountBinding>, -) -> HlsOriginRuntimeNoAccountReason { +fn hls_no_account_reason_for_binding(binding: Option<&HlsOriginAccountBinding>) -> HlsOriginRuntimeNoAccountReason { let Some(binding) = binding else { return HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted; }; @@ -2584,10 +2574,7 @@ fn hls_soft_overlap_delay_ms( return target_duration_ms.saturating_mul(2); } let numerator = origin.saturating_mul(3).saturating_sub(users); - target_duration_ms - .saturating_mul(numerator) - .saturating_add(origin.saturating_sub(1)) - / origin + target_duration_ms.saturating_mul(numerator).saturating_add(origin.saturating_sub(1)) / origin } #[allow(clippy::too_many_arguments, clippy::too_many_lines)] @@ -3086,11 +3073,7 @@ async fn restore_hls_origin_policy_preempt_candidate_reservation( ) { app_state .active_provider - .refresh_provider_reservation( - &candidate.account_name, - &candidate.session_owner, - candidate.reservation_ttl_secs, - ) + .refresh_provider_reservation(&candidate.account_name, &candidate.session_owner, candidate.reservation_ttl_secs) .await; } @@ -3245,8 +3228,7 @@ async fn find_hls_account_overlap_candidate( now_ms: u64, ) -> Option { let sessions = app_state.hls_proxy.sessions().list_sessions().await; - let tuliprox_target_user_connection_capacity = - hls_tuliprox_target_user_connection_capacity(app_state, input).await; + let tuliprox_target_user_connection_capacity = hls_tuliprox_target_user_connection_capacity(app_state, input).await; let origin_input_account_connection_capacity = hls_origin_input_account_connection_capacity(app_state, input).await; let mut speculative_accounts = Vec::new(); for session in &sessions { @@ -3838,8 +3820,9 @@ async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( user: &ProxyUserCredentials, target: &Arc, input: &ConfigInput, - virtual_id: u32, + stream_identity: &HlsEntryStreamIdentity, ) -> bool { + let virtual_id = stream_identity.virtual_id(); let session_token = create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, PlaylistItemType::LiveHls, None); let (connection_admission, _, _) = resolve_playback_request_admission( @@ -3874,7 +3857,7 @@ async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( return false; }; let (shared_hls_session_owner, reservation_ttl_secs) = if hls_cache_enabled_for_target(app_state, target) { - let origin_source = build_hls_origin_source(input, hls_stream_ref_from_virtual_id(virtual_id)); + let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); let reservation_ttl_secs = match app_state.hls_proxy.sessions().get_by_key(&origin_source.session_key()).await { Some(session) => hls_origin_account_reservation_ttl_secs_for_session(&session).await, @@ -3912,14 +3895,14 @@ async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( async fn mark_hls_provisioning_handoff_discontinuity( app_state: &Arc, input: &ConfigInput, - virtual_id: u32, + stream_identity: &HlsEntryStreamIdentity, access_lease_id: Option<&HlsAccessLeaseId>, now_ms: u64, ) -> bool { if !hls_cache_configured(app_state) { return false; } - let origin_source = build_hls_origin_source(input, hls_stream_ref_from_virtual_id(virtual_id)); + let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); let Some(session) = app_state.hls_proxy.sessions().get_by_key(&origin_source.session_key()).await else { return false; }; @@ -3927,7 +3910,7 @@ async fn mark_hls_provisioning_handoff_discontinuity( app_state, &session, input, - virtual_id, + stream_identity.virtual_id(), access_lease_id, now_ms, ) @@ -4026,7 +4009,7 @@ pub(in crate::api) async fn hls_panel_provisioning_poll_manifest_response( user: &ProxyUserCredentials, target: &Arc, input: &ConfigInput, - virtual_id: u32, + stream_identity: &HlsEntryStreamIdentity, original_hls_entry_path: &str, server_path: Option<&str>, ) -> axum::response::Response { @@ -4036,7 +4019,7 @@ pub(in crate::api) async fn hls_panel_provisioning_poll_manifest_response( user, target, input, - virtual_id, + stream_identity, original_hls_entry_path, server_path, HlsProvisioningPollResponseKind::Legacy, @@ -4063,11 +4046,12 @@ async fn hls_panel_provisioning_poll_response( user: &ProxyUserCredentials, target: &Arc, input: &ConfigInput, - virtual_id: u32, + stream_identity: &HlsEntryStreamIdentity, ready_redirect_path: &str, server_path: Option<&str>, response_kind: HlsProvisioningPollResponseKind, ) -> axum::response::Response { + let virtual_id = stream_identity.virtual_id(); let now_ms = current_time_millis(); app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); @@ -4079,14 +4063,14 @@ async fn hls_panel_provisioning_poll_response( user, target, input, - virtual_id, + stream_identity, ) .await { mark_hls_provisioning_handoff_discontinuity( app_state, input, - virtual_id, + stream_identity, response_kind.access_lease_id(), now_ms, ) @@ -4107,13 +4091,15 @@ async fn hls_panel_provisioning_poll_response( }); match status { - HlsProvisioningStatus::Ready | HlsProvisioningStatus::InProgress => hls_custom_video_manifest_response_with_virtual_id( - app_state, - user, - CustomVideoStreamType::Provisioning, - StatusCode::SERVICE_UNAVAILABLE, - Some(virtual_id), - ), + HlsProvisioningStatus::Ready | HlsProvisioningStatus::InProgress => { + hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::Provisioning, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ) + } HlsProvisioningStatus::ProviderExhausted => hls_custom_video_manifest_response_with_virtual_id( app_state, user, @@ -4138,9 +4124,7 @@ async fn hls_panel_provisioning_or_status_response( user, input, virtual_id, - HlsPanelProvisioningRedirectPaths { - waiting_manifest_path: None, - }, + HlsPanelProvisioningRedirectPaths { waiting_manifest_path: None }, server_path, fallback_status, ) @@ -4166,11 +4150,8 @@ fn shared_hls_provisioning_segment_plans( session: &HlsSession, physical_segment_count: usize, ) -> Vec { - let existing_provisioning_segments = session - .segments - .values() - .filter(|entry| is_hls_provisioning_segment(entry)) - .count(); + let existing_provisioning_segments = + session.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); let append_count = if existing_provisioning_segments == 0 { 3 } else { 1 }; let start_proxy_seq = session.proxy_next_seq.unwrap_or(0); (0..append_count) @@ -4228,25 +4209,21 @@ async fn commit_shared_hls_provisioning_segments( for plan in plans { let video = provisioning_segments.get(plan.physical_index)?; let duration_ms = video.duration_ms().unwrap_or(HLS_PROVISIONING_SEGMENT_DURATION_MS); - let metadata = match app_state - .hls_proxy - .segment_cache() - .write_bytes_and_commit(&plan.cache_key, video.as_bytes()) - .await - { - Ok(metadata) => metadata, - Err(err) => { - let safe_session = { - let session_guard = session.read().await; - safe_proxy_session_id(&session_guard.proxy_session_id) - }; - warn!( + let metadata = + match app_state.hls_proxy.segment_cache().write_bytes_and_commit(&plan.cache_key, video.as_bytes()).await { + Ok(metadata) => metadata, + Err(err) => { + let safe_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + warn!( "HLS provisioning segment cache commit failed for shared manifest: session={} seq={} error={err}", safe_session, plan.proxy_seq ); - return None; - } - }; + return None; + } + }; committed.push((plan.clone(), metadata.size, duration_ms)); } Some(committed) @@ -4276,11 +4253,8 @@ async fn ensure_shared_hls_provisioning_handoff_gap( if session_guard.segments.contains_key(&proxy_seq) { return false; } - let existing_provisioning_segments = session_guard - .segments - .values() - .filter(|entry| is_hls_provisioning_segment(entry)) - .count(); + let existing_provisioning_segments = + session_guard.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); SharedHlsProvisioningSegmentPlan { proxy_seq, physical_index: existing_provisioning_segments % provisioning_segments.len(), @@ -4355,9 +4329,10 @@ async fn hls_shared_provisioning_timeline_manifest_response( } session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); - session_guard - .segments - .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + session_guard.segments.insert( + plan.proxy_seq, + shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms), + ); } session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); session_guard.independent_segments = true; @@ -4394,9 +4369,7 @@ async fn hls_shared_provisioning_or_provider_exhausted_response( let now_ms = current_time_millis(); let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); if provisioning_enabled { - app_state - .hls_provisioning - .touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); start_hls_panel_provisioning_once(app_state, input); if let Some(HlsProvisioningStatus::ProviderExhausted) = app_state.hls_provisioning.consumer_status(&input.name, virtual_id, now_ms) @@ -4563,7 +4536,9 @@ async fn prepare_hls_canonical_manifest_origin_runtime( HlsProviderExhaustedResolution::Response(response) => return Err(Box::new(response)), } } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(Box::new(hls_canonical_status_response(status))), + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { + return Err(Box::new(hls_canonical_status_response(status))) + } } } } @@ -4684,8 +4659,7 @@ async fn try_hls_cache_canonical_manifest_response( .await; let manifest_commit_requirement = hls_manifest_commit_requirement(&session, session_outcome, handoff_previous_rendered_at_ms, now_ms).await; - let manifest_boundary_rendered_at_ms = - handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); + let manifest_boundary_rendered_at_ms = handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); let wait_timeout = hls_manifest_wait_timeout_for_requirement(&session, manifest_commit_requirement).await; let cached_manifest_options = hls_cached_manifest_options_for_requirement( wait_timeout, @@ -4954,8 +4928,7 @@ async fn prepare_hls_origin_binding_for_authorized_resource_work( HlsOriginWorkClass::Demand, now_ms, ) - .await - ?; + .await?; if let Some(binding) = prepared_origin.origin_account_binding_to_store { session.write().await.replace_origin_account_binding(Some(binding)); } @@ -5037,8 +5010,7 @@ async fn prepare_hls_transient_origin_io_for_authorized_resource_work( HlsOriginWorkClass::Demand, now_ms, ) - .await - ?; + .await?; if let Some(binding) = prepared_origin.origin_account_binding_to_store { session.write().await.replace_origin_account_binding(Some(binding)); } @@ -5197,18 +5169,19 @@ pub(in crate::api) async fn handle_hls_stream_request( user_session: Option<&UserSession>, hls_url: &str, _archive_reference: Option, - virtual_id: u32, + stream_identity: HlsEntryStreamIdentity, input: &ConfigInput, req_headers: &HeaderMap, connection_permission: UserConnectionPermission, connection_kind: Option, original_hls_entry_path: &str, ) -> impl IntoResponse + Send { + let virtual_id = stream_identity.virtual_id(); if app_state.active_users.is_user_blocked_for_stream(&user.username, virtual_id).await { return axum::http::StatusCode::BAD_REQUEST.into_response(); } - let stream_ref = hls_stream_ref_from_virtual_id(virtual_id); + let stream_ref = stream_identity.stream_ref().to_string(); let normalized_hls_url = normalize_xtream_live_hls_url(hls_url, input); if normalized_hls_url != hls_url { debug_if_enabled!( @@ -5268,15 +5241,9 @@ pub(in crate::api) async fn handle_hls_stream_request( false, connection_priority_for_kind( user, - session - .connection_kind - .or(connection_kind) - .unwrap_or(crate::api::model::ConnectionKind::Normal), + session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal), ), - session - .connection_kind - .or(connection_kind) - .unwrap_or(crate::api::model::ConnectionKind::Normal), + session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal), Some(session.token.as_str()), ) .await @@ -5343,7 +5310,7 @@ pub(in crate::api) async fn handle_hls_stream_request( None, ); let hls_session_owner = if hls_cache_enabled_for_target(app_state, target) { - let session_key = HlsSessionKey::new(input.id, virtual_id.to_string()); + let session_key = HlsSessionKey::new(input.id, stream_identity.stream_ref()); let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); Some(build_hls_origin_session_owner(&proxy_session_id)) } else { @@ -5398,27 +5365,7 @@ pub(in crate::api) async fn handle_hls_stream_request( app_state.connection_manager.release_provider_handle(provider_handle).await; let input_source = InputSource::from(input).with_url(request_url); - let use_manual_redirects = app_state.should_use_manual_redirects(); - let download_result = if use_manual_redirects { - request::download_text_content_with_manual_redirects_and_headers( - &app_state.app_config, - &app_state.http_client_no_redirect.load(), - &input_source, - Some(&headers), - false, - MAX_MANUAL_REDIRECTS, - ) - .await - } else { - request::download_text_content_with_headers( - &app_state.app_config, - &app_state.http_client.load(), - &input_source, - Some(&headers), - false, - ) - .await - }; + let download_result = download_legacy_hls_manifest(app_state, &input_source, &headers).await; match download_result { Ok((content, response_url, response_headers)) => { let encrypt_secret = app_state.get_encrypt_secret(); @@ -5448,7 +5395,7 @@ pub(in crate::api) async fn handle_hls_stream_request( hls_response(hls_content).into_response() } Err(err) => { - error!("Failed to download m3u8 {}", sanitize_sensitive_info(&err.to_string())); + error!("Failed to download m3u8: {}", request::text_response_error_log_label(&err)); if let Some(session_token) = session_token.as_deref() { terminate_failed_hls_manifest_session(app_state, &user.username, session_token).await; } @@ -5477,13 +5424,39 @@ async fn get_stream_channel( m3u_get_item_for_stream_id(virtual_id, app_state, target).await.ok().map(|pli| pli.to_stream_channel(target_id)) } -pub(in crate::api) async fn resolve_hls_virtual_input_for_target( +fn hls_stream_identity_or_unavailable( + item: &impl PlaylistEntry, + virtual_id: u32, +) -> Result { + HlsEntryStreamIdentity::from_playlist_item(item).ok_or_else(|| { + warn!("HLS input stream identity missing for virtual_id={virtual_id}; refresh target playlist"); + StatusCode::SERVICE_UNAVAILABLE + }) +} + +pub(in crate::api) async fn resolve_hls_virtual_source_for_target( app_state: &Arc, target: &Arc, virtual_id: u32, -) -> Option> { - let channel = get_stream_channel(app_state, target, virtual_id).await?; - app_state.app_config.get_input_by_name(&channel.input_name) +) -> Result { + let (input_name, stream_identity) = if target.has_output(TargetType::Xtream) { + if let Ok(item) = xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await { + let stream_identity = hls_stream_identity_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_identity) + } else { + let item = + m3u_get_item_for_stream_id(virtual_id, app_state, target).await.map_err(|_| StatusCode::NOT_FOUND)?; + let stream_identity = hls_stream_identity_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_identity) + } + } else { + let item = + m3u_get_item_for_stream_id(virtual_id, app_state, target).await.map_err(|_| StatusCode::NOT_FOUND)?; + let stream_identity = hls_stream_identity_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_identity) + }; + let input = app_state.app_config.get_input_by_name(&input_name).ok_or(StatusCode::NOT_FOUND)?; + Ok(HlsResolvedVirtualSource { input, stream_identity }) } async fn resolve_hls_origin_playlist_url( @@ -5622,10 +5595,8 @@ async fn hls_proxy_manifest( let proxy_session_id = ProxySessionId(params.proxy_session_id); let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id); let now_ms = current_time_millis(); - let access_lease_snapshot = app_state - .hls_proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) - .await; + let access_lease_snapshot = + app_state.hls_proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await; if let Some(session) = app_state.hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await { app_state .hls_proxy @@ -5837,6 +5808,7 @@ async fn hls_api_stream_resolved( input: &input, user: &user, session_reservation_ttl_secs: get_hls_session_ttl_secs(&app_state), + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, }, None, ) @@ -5893,6 +5865,17 @@ async fn hls_api_stream_resolved( let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); if is_hls_url(&session.stream_url) { + let source = match resolve_hls_virtual_source_for_target(&app_state, &target, virtual_id).await { + Ok(source) if source.input.id == input.id => source, + Ok(source) => { + warn!( + "HLS input context mismatch for virtual_id={virtual_id}: expected_input_id={}, resolved_input_id={}", + input.id, source.input.id + ); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + Err(status) => return status.into_response(), + }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); return handle_hls_stream_request( &fingerprint, @@ -5902,7 +5885,7 @@ async fn hls_api_stream_resolved( Some(session), &session.stream_url, archive_reference, - virtual_id, + source.stream_identity, &input, &req_headers, connection_permission, @@ -5946,6 +5929,7 @@ async fn hls_api_stream_resolved( input: &input, user: &user, session_reservation_ttl_secs: get_hls_session_ttl_secs(&app_state), + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, }, grace_mode, ) @@ -5986,31 +5970,32 @@ pub fn hls_api_register() -> axum::Router> { mod tests { use super::{ build_hls_manifest_request_headers, extract_hls_provider_session_headers, hls_api_register, - m3u_archive_epg_reference_ts, + m3u_archive_epg_reference_ts, MAX_HLS_MANIFEST_BYTES, }; use crate::{ api::model::{ begin_hls_origin_account_io, build_hls_custom_video_manifest_body, build_proxy_session_id, build_transient_resource_id, finish_hls_origin_account_io, ActiveProviderManager, ActiveUserManager, AppState, CacheAccessState, CancelTokens, ConnectionKind, ConnectionManager, CreateUserSessionParams, - CustomVideoStreamType, EventManager, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, - HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, + CustomVideoStreamType, EventManager, HlsAccessContext, HlsAccessLease, + HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, HlsAccessLeaseTiming, HlsAccessLeaseValidationError, HlsEffectiveOriginAcquirePolicy, HlsFreshManifestRequiredReason, HlsLifecycleEvent, HlsLifecycleEventKey, HlsManifestCommitRequirement, HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginIoContext, - HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentFile, HlsSessionHandle, - HlsSession, HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, ManualPlaylistUpdateRequest, + HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentFile, HlsSession, + HlsSessionHandle, HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, ManualPlaylistUpdateRequest, MapCacheStatus, MapEntry, MetadataUpdateManager, OriginMapKey, OriginSegmentFetchRef, OriginSegmentKey, PlaybackLifecycle, PlaylistStorageState, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, ProxyMapId, ProxySessionId, RenderedManifest, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, - SegmentFetchPriority, SharedStreamManager, TransientResourceKind, TransientResourceRef, TransportStreamBuffer, - UpdateGuard, UserSession, + SegmentFetchPriority, SharedStreamManager, TransientObjectCacheKey, TransientObjectCacheStatus, + TransientResourceId, TransientResourceKind, TransientResourceRef, TransportStreamBuffer, UpdateGuard, + UserSession, }, auth::Fingerprint, model::{ - ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigProvider, ConfigTarget, HlsCacheConfig, - ConfigSource, CustomStreamResponse, ProcessTargets, ProxyUserCredentials, ReverseProxyConfig, - ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, TargetUser, + ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigProvider, ConfigSource, + ConfigTarget, CustomStreamResponse, HlsCacheConfig, ProcessTargets, ProxyUserCredentials, + ReverseProxyConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, TargetUser, }, processing::parser::hls::{ origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, @@ -6026,7 +6011,15 @@ mod tests { response::IntoResponse, }; use http_body_util::BodyExt; - use shared::model::{ConfigPaths, ConfigProviderDto, ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, HlsCacheConfigDto, HlsSegmentRepairMode, HlsStripMode, InputType, PlaylistItemType, ProviderUrlSelectionPolicy, ReverseProxyConfigDto, StreamConfigDto, TargetOutputDto, UserConnectionPermission, XtreamTargetOutputDto}; + use shared::{ + model::{ + ConfigPaths, ConfigProviderDto, ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, + HlsCacheConfigDto, HlsSegmentRepairMode, HlsStripMode, InputType, M3uPlaylistItem, M3uTargetOutputDto, + PlaylistItem, PlaylistItemHeader, PlaylistItemType, ProviderUrlSelectionPolicy, ReverseProxyConfigDto, + StreamConfigDto, TargetOutputDto, UserConnectionPermission, XtreamCluster, XtreamTargetOutputDto, + }, + utils::Internable, + }; use std::{collections::HashMap, fmt::Write as _, net::SocketAddr, sync::Arc, time::Duration}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, @@ -6145,6 +6138,44 @@ mod tests { }) } + fn test_m3u_hls_share_target() -> ConfigTarget { + ConfigTarget::from(&ConfigTargetDto { + id: 2, + name: "m3u-target".to_string(), + options: Some(ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, + ..Default::default() + }), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())], + use_memory_cache: true, + ..Default::default() + }) + } + + fn test_m3u_hls_item(input: &ConfigInput, virtual_id: u32, input_stream_id: &str, url: &str) -> M3uPlaylistItem { + M3uPlaylistItem::from(&PlaylistItem { + header: PlaylistItemHeader { + id: input_stream_id.intern(), + virtual_id, + input_name: Arc::clone(&input.name), + url: url.intern(), + item_type: PlaylistItemType::LiveHls, + xtream_cluster: XtreamCluster::Live, + input_stream_id: input_stream_id.intern(), + ..PlaylistItemHeader::default() + }, + }) + } + + async fn cache_test_m3u_hls_item(app_state: &Arc, target: &ConfigTarget, item: M3uPlaylistItem) { + let mut playlist = crate::repository::BPlusTree::new(); + playlist.insert(item.virtual_id, item); + app_state + .playlists + .cache_playlist(&target.name, crate::api::model::PlaylistStorage::M3uPlaylist(Box::new(playlist))) + .await; + } + #[allow(dead_code)] fn test_hls_input() -> ConfigInput { ConfigInput { @@ -6167,10 +6198,7 @@ mod tests { batch_files: vec![], provider: vec![], inputs: vec![Arc::clone(&input)], - sources: vec![ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![Arc::new(target)], - }], + sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::new(target)] }], templates: None, })); } @@ -6298,14 +6326,10 @@ mod tests { ); assert!(manifest.contains("#EXT-X-ENDLIST")); - assert!(manifest.contains( - "https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts" - )); + assert!(manifest.contains("https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts")); assert_eq!(manifest.matches("#EXTINF:10.0,").count(), 12); assert_eq!( - manifest - .matches("https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts") - .count(), + manifest.matches("https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts").count(), 12 ); } @@ -6344,10 +6368,11 @@ mod tests { } #[test] - fn hls_response_uses_rfc8216_manifest_content_type() { + fn hls_response_uses_rfc8216_content_type_and_remains_tower_compressible() { let response = super::hls_response("#EXTM3U\n".to_string()).into_response(); assert_eq!(response.headers().get(header::CONTENT_TYPE).unwrap(), "application/vnd.apple.mpegurl"); + assert!(crate::api::api_utils::should_compress_response(&response)); } #[test] @@ -6375,11 +6400,13 @@ mod tests { let mut input_headers = HashMap::new(); input_headers.insert("User-Agent".to_string(), "Input-UA".to_string()); input_headers.insert("Accept-Language".to_string(), "de".to_string()); + input_headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); input_headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); input_headers.insert("X-Origin-Secret".to_string(), "input-secret".to_string()); let mut request_headers = HeaderMap::new(); request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); request_headers.insert(header::USER_AGENT, HeaderValue::from_static("Client-UA")); request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer client-secret")); request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); @@ -6400,6 +6427,7 @@ mod tests { assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Input-UA"); assert_eq!(headers.get("accept-language").expect("accept language"), "de"); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("accept encoding"), "identity"); assert!(!headers.contains_key(header::RANGE)); assert!(!headers.contains_key(header::AUTHORIZATION)); assert!(!headers.contains_key(header::COOKIE)); @@ -6410,6 +6438,116 @@ mod tests { assert!(!headers.contains_key("cf-ray")); } + #[tokio::test] + async fn legacy_hls_manifest_decodes_supported_origin_codings_and_enforces_identity() { + const MANIFEST: &str = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nsegment.ts\n"; + + for coding in ["gzip", "deflate", "br", "zstd"] { + let encoded = encode_test_manifest(coding, MANIFEST.as_bytes()).await; + let origin = spawn_test_encoded_manifest_origin(Some(coding), encoded, Duration::ZERO).await; + let input = legacy_manifest_test_input(&origin); + let client_headers = legacy_manifest_test_client_headers(); + + let (manifest, final_url, _) = + super::download_legacy_hls_manifest(&test_app_state(), &input, &client_headers) + .await + .unwrap_or_else(|error| panic!("{coding} manifest should decode: {error}")); + + assert_eq!(manifest, MANIFEST, "coding={coding}"); + assert_eq!(final_url, input.url, "coding={coding}"); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1, "coding={coding}"); + assert!( + requests[0].to_ascii_lowercase().contains("\r\naccept-encoding: identity\r\n"), + "coding={coding}, request={}", + requests[0] + ); + } + } + + #[tokio::test] + async fn legacy_hls_manifest_handles_identity_and_headerless_gzip_magic() { + const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-TARGETDURATION:4\n"; + let cases = [("identity", MANIFEST.to_vec()), ("gzip-magic", encode_test_manifest("gzip", MANIFEST).await)]; + + for (case, body) in cases { + let origin = spawn_test_encoded_manifest_origin(None, body, Duration::ZERO).await; + let input = legacy_manifest_test_input(&origin); + + let (manifest, _, _) = + super::download_legacy_hls_manifest(&test_app_state(), &input, &legacy_manifest_test_client_headers()) + .await + .unwrap_or_else(|error| panic!("{case} manifest should decode: {error}")); + + assert_eq!(manifest.as_bytes(), MANIFEST, "case={case}"); + } + } + + #[tokio::test] + async fn legacy_hls_manifest_limit_applies_after_decompression() { + let decoded = vec![b'x'; MAX_HLS_MANIFEST_BYTES + 1]; + let origin = spawn_test_encoded_manifest_origin( + Some("gzip"), + encode_test_manifest("gzip", &decoded).await, + Duration::ZERO, + ) + .await; + let input = legacy_manifest_test_input(&origin); + + let error = + super::download_legacy_hls_manifest(&test_app_state(), &input, &legacy_manifest_test_client_headers()) + .await + .expect_err("decoded manifest above limit must fail"); + + assert!(matches!( + error.get_ref().and_then(|source| source.downcast_ref()), + Some(crate::utils::content_coding::ContentBodyReadError::LimitExceeded { limit }) + if *limit == MAX_HLS_MANIFEST_BYTES + )); + } + + #[tokio::test] + async fn legacy_hls_manifest_deadline_includes_full_body_read() { + let origin = spawn_test_encoded_manifest_origin(None, b"#EXTM3U\n".to_vec(), Duration::from_millis(100)).await; + let input = legacy_manifest_test_input(&origin); + let hls_config = + HlsCacheConfig::from(&HlsCacheConfigDto { origin_manifest_timeout_ms: 10, ..Default::default() }); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + + let error = super::download_legacy_hls_manifest(&app_state, &input, &legacy_manifest_test_client_headers()) + .await + .expect_err("complete manifest body read must honor the deadline"); + + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + } + + #[tokio::test] + async fn legacy_hls_manifest_distinguishes_invalid_utf8_from_decoder_failure() { + let invalid_utf8_origin = spawn_test_encoded_manifest_origin(None, vec![0xff], Duration::ZERO).await; + let invalid_utf8_input = legacy_manifest_test_input(&invalid_utf8_origin); + let invalid_utf8 = super::download_legacy_hls_manifest( + &test_app_state(), + &invalid_utf8_input, + &legacy_manifest_test_client_headers(), + ) + .await + .expect_err("invalid UTF-8 must fail"); + + let corrupt_origin = + spawn_test_encoded_manifest_origin(Some("gzip"), vec![0x1f, 0x8b, 0x08, 0x00], Duration::ZERO).await; + let corrupt_input = legacy_manifest_test_input(&corrupt_origin); + let decoder_failure = super::download_legacy_hls_manifest( + &test_app_state(), + &corrupt_input, + &legacy_manifest_test_client_headers(), + ) + .await + .expect_err("corrupt gzip must fail"); + + assert_eq!(invalid_utf8.kind(), std::io::ErrorKind::InvalidData); + assert!(crate::utils::content_coding::content_decoding_error_from_io(&decoder_failure).is_some()); + } + #[test] fn hls_proxy_public_path_prefix_rewrites_only_proxy_hls_uri_surfaces() { let body = concat!( @@ -6472,7 +6610,10 @@ mod tests { .expect("provider entry url should resolve"); assert_eq!(origin.session_entry_url.as_str(), "provider://demo/live/account-a/token-a/1025130.m3u8"); - assert_eq!(origin.session_entry_url.url_failover_provider().expect("provider failover config").name.as_ref(), "demo"); + assert_eq!( + origin.session_entry_url.url_failover_provider().expect("provider failover config").name.as_ref(), + "demo" + ); let provider_key = super::build_hls_origin_source(&input, "1025130").session_key(); let direct_key = super::build_hls_origin_source(&input, "1025130").session_key(); assert_eq!(provider_key, direct_key); @@ -6585,7 +6726,10 @@ mod tests { let failover_key = super::build_hls_origin_source(&input, "80510").session_key(); let direct_key = super::build_hls_origin_source(&input, "80510").session_key(); - assert_eq!(origin.session_entry_url.as_str(), "provider://mirror-group/live/source-user/source-pass/1025126.m3u8"); + assert_eq!( + origin.session_entry_url.as_str(), + "provider://mirror-group/live/source-user/source-pass/1025126.m3u8" + ); assert!(origin.session_entry_url.url_failover_provider().is_some()); assert_eq!(failover_key, direct_key); assert!(!failover_key.stable_value().contains("provider://")); @@ -6714,10 +6858,7 @@ mod tests { None, ) .expect("provider scheme fetch url should remain failover capable"); - assert_eq!( - provider_scheme_without_account_rewrite, - "provider://demo/live/source-user/source-pass/12345.m3u8" - ); + assert_eq!(provider_scheme_without_account_rewrite, "provider://demo/live/source-user/source-pass/12345.m3u8"); let provider_scheme_fetch_url = super::build_hls_origin_fetch_url( &provider_scheme_input, @@ -6727,10 +6868,7 @@ mod tests { ) .expect("provider scheme fetch url should use selected runtime account without losing failover"); - assert_eq!( - provider_scheme_fetch_url, - "provider://demo/live/provider-user/provider-pass/12345.m3u8" - ); + assert_eq!(provider_scheme_fetch_url, "provider://demo/live/provider-user/provider-pass/12345.m3u8"); let failover_context = super::hls_url_failover_provider_for_origin_context( &provider_scheme_input, "provider://demo/live/source-user/source-pass/12345.m3u8", @@ -7118,10 +7256,10 @@ mod tests { fn test_app_state() -> Arc { test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::new())) } fn disable_custom_stream_response(app_state: &Arc) { - app_state.app_config.config.store(Arc::new(Config { - custom_stream_response_enabled: false, - ..Default::default() - })); + app_state + .app_config + .config + .store(Arc::new(Config { custom_stream_response_enabled: false, ..Default::default() })); } fn enable_provider_exhausted_custom_response(app_state: &Arc) { @@ -7141,10 +7279,10 @@ mod tests { fn enable_low_priority_preempted_custom_response(app_state: &Arc) { let config = app_state.app_config.config.load(); - app_state.app_config.config.store(Arc::new(Config { - custom_stream_response_enabled: true, - ..config.as_ref().clone() - })); + app_state + .app_config + .config + .store(Arc::new(Config { custom_stream_response_enabled: true, ..config.as_ref().clone() })); let mut ts_packet = vec![0_u8; 188]; ts_packet[0] = 0x47; app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { @@ -7161,10 +7299,10 @@ mod tests { fn enable_channel_unavailable_custom_response(app_state: &Arc) { let config = app_state.app_config.config.load(); - app_state.app_config.config.store(Arc::new(Config { - custom_stream_response_enabled: true, - ..config.as_ref().clone() - })); + app_state + .app_config + .config + .store(Arc::new(Config { custom_stream_response_enabled: true, ..config.as_ref().clone() })); let mut ts_packet = vec![0_u8; 188]; ts_packet[0] = 0x47; app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { @@ -7545,14 +7683,12 @@ mod tests { super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; - assert!(app_state - .hls_proxy - .is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000) - .await); - assert!(!app_state - .hls_proxy - .is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000) - .await); + assert!( + app_state.hls_proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await + ); + assert!( + !app_state.hls_proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await + ); let loser_binding_mode = loser.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); assert!(matches!( loser_binding_mode, @@ -7642,7 +7778,8 @@ mod tests { .await .expect("authorized origin work can reacquire a detached binding"); - let binding = prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); + let binding = + prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); assert_eq!(binding.account_name.as_ref(), "account-a"); assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); @@ -7685,7 +7822,8 @@ mod tests { .await .expect("interactive work should use soft-active overlap before grace"); - let binding = prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); + let binding = + prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); assert_eq!(binding.account_name, input.name); assert!(matches!( &binding.binding_mode, @@ -7770,7 +7908,8 @@ mod tests { async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal_binding() { let input = single_hls_provider_input("policy-no-preempt-input"); let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let normal_session = create_bound_hls_test_session(&app_state, &input, "normal", input.name.as_ref(), 1_000).await; + let normal_session = + create_bound_hls_test_session(&app_state, &input, "normal", input.name.as_ref(), 1_000).await; { let mut session = normal_session.write().await; session.target_duration = Some(10); @@ -8021,10 +8160,7 @@ mod tests { .expect("provisioning manifest should render local segments"); { let session_guard = session.read().await; - assert!(session_guard - .segments - .values() - .any(crate::api::model::is_hls_provisioning_segment)); + assert!(session_guard.segments.values().any(crate::api::model::is_hls_provisioning_segment)); assert_eq!(session_guard.segments.len(), 3); assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); } @@ -8046,6 +8182,24 @@ mod tests { assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); } + #[tokio::test] + async fn provisioning_handoff_finds_shared_session_by_input_stream_id_not_virtual_id() { + let input = single_hls_provider_input("origin-id-handoff-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + let origin_session = create_unbound_hls_test_session(&app_state, &input, "80510", 1_000).await; + let virtual_id_session = create_unbound_hls_test_session(&app_state, &input, "1001", 1_000).await; + let stream_identity = super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); + + assert!( + super::mark_hls_provisioning_handoff_discontinuity(&app_state, &input, &stream_identity, None, 2_000,) + .await + ); + + assert!(origin_session.read().await.pending_handoff_discontinuity_sequence.is_some()); + assert_eq!(virtual_id_session.read().await.pending_handoff_discontinuity_sequence, None); + } + #[tokio::test] async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segments() { let input = single_hls_provider_input("shared-provisioning-handoff-input"); @@ -8163,12 +8317,8 @@ mod tests { #[test] fn hls_detached_origin_binding_reclaimed_by_owner_maps_to_preempted_no_account_reason() { let proxy_session_id = ProxySessionId("preempted-session".to_string()); - let mut binding = HlsOriginAccountBinding::new( - Arc::from("input-a"), - Arc::from("account-a"), - &proxy_session_id, - 1_000, - ); + let mut binding = + HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, 2_000); assert_eq!( @@ -8180,12 +8330,8 @@ mod tests { #[test] fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_reason() { let proxy_session_id = ProxySessionId("soft-window-session".to_string()); - let mut binding = HlsOriginAccountBinding::new( - Arc::from("input-a"), - Arc::from("account-a"), - &proxy_session_id, - 1_000, - ); + let mut binding = + HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, 2_000); assert_eq!( @@ -8217,10 +8363,10 @@ mod tests { } #[tokio::test] - async fn hls_hard_manifest_failure_forces_next_fresh_commit() { - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; + async fn hls_hard_manifest_failure_forces_next_fresh_commit() { + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; session.last_rendered_manifest = Some(RenderedManifest { body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), first_proxy_seq: 1, @@ -8239,79 +8385,79 @@ mod tests { HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure } - ); + ); + } + + #[tokio::test] + async fn hls_normal_expired_session_allows_committed_manifest_while_manifest_valid() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_add(10_000), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(1_000), + segment_proxy_seqs: vec![1], + }); } - #[tokio::test] - async fn hls_normal_expired_session_allows_committed_manifest_while_manifest_valid() { - let now_ms = 100_000; - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 1, - playlist_duration_ms: 4_000, - valid_until_ms: now_ms.saturating_add(10_000), - render_gap_segments: 0, - rendered_at_ms: now_ms.saturating_sub(1_000), - segment_proxy_seqs: vec![1], - }); + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::CommittedManifestAllowed + ); + } + + #[tokio::test] + async fn hls_normal_expired_session_requires_fresh_commit_after_manifest_validity() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_sub(1), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(10_000), + segment_proxy_seqs: vec![1], + }); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation } + ); + } - assert_eq!( - super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, - HlsManifestCommitRequirement::CommittedManifestAllowed - ); - } - - #[tokio::test] - async fn hls_normal_expired_session_requires_fresh_commit_after_manifest_validity() { - let now_ms = 100_000; - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 1, - playlist_duration_ms: 4_000, - valid_until_ms: now_ms.saturating_sub(1), - render_gap_segments: 0, - rendered_at_ms: now_ms.saturating_sub(10_000), - segment_proxy_seqs: vec![1], - }); - } - - assert_eq!( - super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, - HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ExpiredRevalidation - } - ); - } - - #[tokio::test] - async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { + #[tokio::test] + async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { let app_state = test_app_state(); let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; @@ -8804,7 +8950,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -8874,7 +9020,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -8929,7 +9075,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -8981,7 +9127,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -9049,10 +9195,7 @@ mod tests { .expect("hls cache should handle valid live hls entrypoint"); assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); - assert!( - response.headers().get(header::RETRY_AFTER).is_none(), - "custom response must not expose retry-after" - ); + assert!(response.headers().get(header::RETRY_AFTER).is_none(), "custom response must not expose retry-after"); let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); @@ -9133,7 +9276,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -9197,7 +9340,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -9252,7 +9395,7 @@ mod tests { origin_source, }, HeaderMap::new(), - None, + None, "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -9378,7 +9521,7 @@ mod tests { user.password = "hls-pass".to_string(); let input = test_hls_input(); let target = test_hls_share_target(true); - let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 1001); let response = super::handle_hls_stream_request( &test_fingerprint(), @@ -9386,9 +9529,9 @@ mod tests { &user, &target, None, - "http://origin.example.com/live/user/pass/12345.m3u8", + "http://origin.example.com/live/user/pass/1001.m3u8", None, - 12345, + super::HlsEntryStreamIdentity::new(1001, "80510").expect("valid input stream identity"), &input, &HeaderMap::new(), UserConnectionPermission::Allowed, @@ -9404,6 +9547,128 @@ mod tests { assert!(location.starts_with("/hls/shared/live/")); assert!(location.ends_with("/manifest.m3u8")); assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 1); + + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(location).to_string()); + let origin_key = HlsSessionKey::new(input.id, "80510"); + let virtual_id_key = HlsSessionKey::new(input.id, "1001"); + assert_eq!(origin_key.stable_value(), "input:1|hls|80510"); + assert_eq!(proxy_session_id, build_proxy_session_id(&origin_key, &app_state.get_encrypt_secret())); + assert_ne!(proxy_session_id, build_proxy_session_id(&virtual_id_key, &app_state.get_encrypt_secret())); + + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(location).to_string()); + let lease = app_state + .hls_proxy + .access_leases() + .write() + .await + .response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) + .expect("access lease"); + assert_eq!(lease.stream_ref, "80510"); + assert_eq!(lease.virtual_id, 1001); + } + + #[tokio::test] + async fn hls_cache_entry_shares_content_session_across_targets_but_keeps_distinct_virtual_leases() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let first_target = test_hls_share_target(true); + let mut second_target = test_hls_share_target(true); + second_target.id = 2; + + let first_response = super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &first_target, + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + super::HlsEntryStreamIdentity::new(1001, "80510").expect("first input stream identity"), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::build_virtual_hls_entry_path(&first_target, &input, &user, 1001), + ) + .await + .into_response(); + let second_response = super::handle_hls_stream_request( + &test_fingerprint_with_addr(test_addr_with_port(55124)), + &app_state, + &user, + &second_target, + None, + "http://origin.example.com/live/user/pass/9007.m3u8", + None, + super::HlsEntryStreamIdentity::new(9007, "80510").expect("second input stream identity"), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::build_virtual_hls_entry_path(&second_target, &input, &user, 9007), + ) + .await + .into_response(); + + assert_eq!(first_response.status(), StatusCode::TEMPORARY_REDIRECT); + assert_eq!(second_response.status(), StatusCode::TEMPORARY_REDIRECT); + let first_location = first_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("first location header"); + let second_location = second_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("second location header"); + assert_eq!( + proxy_session_id_from_redirect_location(first_location), + proxy_session_id_from_redirect_location(second_location) + ); + assert_ne!( + access_lease_id_from_redirect_location(first_location), + access_lease_id_from_redirect_location(second_location) + ); + + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(first_location).to_string()); + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(first_location).to_string()); + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(second_location).to_string()); + let now_ms = super::current_time_millis(); + let mut leases = app_state.hls_proxy.access_leases().write().await; + let first_lease = + leases.response_snapshot(&first_lease_id, &proxy_session_id, now_ms).expect("first access lease"); + let second_lease = + leases.response_snapshot(&second_lease_id, &proxy_session_id, now_ms).expect("second access lease"); + assert_eq!(first_lease.virtual_id, 1001); + assert_eq!(second_lease.virtual_id, 9007); + assert_eq!(first_lease.stream_ref, "80510"); + assert_eq!(second_lease.stream_ref, "80510"); + } + + #[tokio::test] + async fn hls_virtual_source_resolver_rejects_missing_input_stream_id_with_service_unavailable() { + let input = single_hls_provider_input("missing-origin-id-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let target = Arc::new(test_m3u_hls_share_target()); + let item = test_m3u_hls_item( + &input, + 1001, + "", + "http://account.example.com/live/account-user/account-pass/channel.m3u8", + ); + cache_test_m3u_hls_item(&app_state, &target, item).await; + + let status = super::resolve_hls_virtual_source_for_target(&app_state, &target, 1001) + .await + .expect_err("missing input stream identity must fail safely"); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); } #[tokio::test] @@ -9426,7 +9691,7 @@ mod tests { None, "http://origin.example.com/live/user/pass/12345.m3u8", None, - 12345, + super::HlsEntryStreamIdentity::new(12345, "80510").expect("valid input stream identity"), &input, &HeaderMap::new(), UserConnectionPermission::Allowed, @@ -9723,6 +9988,53 @@ mod tests { app_state.connection_manager.release_provider_handle(same_owner_handle).await; } + #[tokio::test] + async fn hls_virtual_entry_reservation_uses_input_stream_id_for_shared_session_owner() { + let input = single_hls_provider_input("origin-id-reservation-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + let target = Arc::new(test_m3u_hls_share_target()); + let item = test_m3u_hls_item( + &input, + 1001, + "80510", + "http://account.example.com/live/account-user/account-pass/80510.m3u8", + ); + cache_test_m3u_hls_item(&app_state, &target, item).await; + let stream_identity = super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + + assert!( + super::try_reserve_hls_virtual_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &target, + &input, + &stream_identity, + ) + .await + ); + + let expected_key = HlsSessionKey::new(input.id, "80510"); + let expected_proxy_session_id = build_proxy_session_id(&expected_key, &app_state.get_encrypt_secret()); + let expected_owner = crate::api::model::build_hls_origin_session_owner(&expected_proxy_session_id); + let same_owner_handle = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55253), + false, + 0, + ConnectionKind::Normal, + Some(&expected_owner), + ) + .await; + assert!(same_owner_handle.is_some(), "reservation must be owned by input:1|hls|80510, not virtual_id=1001"); + app_state.connection_manager.release_provider_handle(same_owner_handle).await; + } + #[tokio::test] async fn hls_cache_entry_creates_new_lease_for_same_user_session_and_proxy_session() { let app_state = test_app_state(); @@ -10216,12 +10528,12 @@ mod tests { .hls_proxy .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) .await; - let proxy_session_id = { - let mut session = session.write().await; - session.origin_refresh.next_fetch_allowed_at_ms = u64::MAX; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - session.last_rendered_manifest = Some(RenderedManifest { + let proxy_session_id = { + let mut session = session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = u64::MAX; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + session.last_rendered_manifest = Some(RenderedManifest { body: format!( "#EXTM3U\n#EXT-X-MAP:URI=\"/hls/shared/live/{proxy_session_id}/{}/map/000000.mp4\"\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_id}/{}/000123.ts\n", crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, @@ -10235,8 +10547,8 @@ mod tests { rendered_at_ms, segment_proxy_seqs: vec![123], }); - proxy_session_id - }; + proxy_session_id + }; let proxy_session_id = ProxySessionId(proxy_session_id); let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); @@ -10257,7 +10569,7 @@ mod tests { origin_source: super::build_hls_origin_source(&input, "12345"), }, HeaderMap::new(), - Some("/iptv"), + Some("/iptv"), "/live/hls-user/hls-pass/12345.m3u8", ) .await @@ -10383,10 +10695,7 @@ mod tests { assert!(!body.contains("/000003.ts")); assert!(body.contains(&access_lease_id.0)); assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!( - session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, - stored_before - ); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); assert_eq!(media_uri_count(&stored_before), 6); } @@ -10431,10 +10740,7 @@ mod tests { assert!(!body.contains("/000003.ts")); assert!(body.contains(&access_lease_id.0)); assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!( - session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, - stored_before - ); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); assert_eq!(media_uri_count(&stored_before), 6); } @@ -10705,15 +11011,17 @@ mod tests { .await; let _proxy_session_id = { let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - session - .transient - .replace_manifest_with_validity(transient_manifest_body(&proxy_session_id), rendered_at_ms, 60_000); - proxy_session_id - }; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_id), + rendered_at_ms, + 60_000, + ); + proxy_session_id + }; let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; @@ -10916,11 +11224,11 @@ mod tests { }; let session_for_commit = Arc::clone(&session); let proxy_session_for_body = proxy_session_id.0.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - session.last_rendered_manifest = Some(RenderedManifest { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session.last_rendered_manifest = Some(RenderedManifest { body: format!( "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER @@ -10933,8 +11241,8 @@ mod tests { rendered_at_ms, segment_proxy_seqs: vec![100], }); - session.origin_refresh.in_flight = false; - }); + session.origin_refresh.in_flight = false; + }); let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; @@ -10973,14 +11281,16 @@ mod tests { let session_for_commit = Arc::clone(&session); let proxy_session_for_body = proxy_session_id.0.clone(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - session - .transient - .replace_manifest_with_validity(transient_manifest_body(&proxy_session_for_body), rendered_at_ms, 60_000); - session.origin_refresh.in_flight = false; - }); + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_for_body), + rendered_at_ms, + 60_000, + ); + session.origin_refresh.in_flight = false; + }); let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; @@ -11021,14 +11331,16 @@ mod tests { let session_for_commit = Arc::clone(&session); let proxy_session_for_body = proxy_session_id.0.clone(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - session - .transient - .replace_manifest_with_validity(transient_manifest_body(&proxy_session_for_body), rendered_at_ms, 60_000); - session.origin_refresh.in_flight = false; - }); + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_for_body), + rendered_at_ms, + 60_000, + ); + session.origin_refresh.in_flight = false; + }); let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; @@ -11553,6 +11865,114 @@ mod tests { TestSegmentOrigin { base_url: format!("http://{addr}"), task } } + struct TestEncodedManifestOrigin { + base_url: String, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for TestEncodedManifestOrigin { + fn drop(&mut self) { self.task.abort(); } + } + + async fn spawn_test_encoded_manifest_origin( + content_encoding: Option<&'static str>, + body: Vec, + body_delay: Duration, + ) -> TestEncodedManifestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let body = Arc::new(body); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let body = Arc::clone(&body); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); + if let Some(content_encoding) = content_encoding { + let _ = writeln!(&mut response, "Content-Encoding: {content_encoding}\r"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + if body_delay.is_zero() { + let _ = socket.write_all(body.as_slice()).await; + } else { + let split_at = body.len().min(4); + let _ = socket.write_all(&body[..split_at]).await; + tokio::time::sleep(body_delay).await; + let _ = socket.write_all(&body[split_at..]).await; + } + }); + } + }); + TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task } + } + + fn legacy_manifest_test_input(origin: &TestEncodedManifestOrigin) -> crate::model::InputSource { + crate::model::InputSource { + name: Arc::from("legacy-content-coding-test"), + url: format!("{}/manifest.m3u8", origin.base_url), + provider: None, + username: None, + password: None, + method: shared::model::InputFetchMethod::GET, + headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), + } + } + + fn legacy_manifest_test_client_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + headers + } + + async fn encode_test_manifest(content_encoding: &str, body: &[u8]) -> Vec { + match content_encoding { + "gzip" => { + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("gzip test body encodes"); + encoder.shutdown().await.expect("gzip test encoder finishes"); + encoder.into_inner() + } + "deflate" => { + let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("deflate test body encodes"); + encoder.shutdown().await.expect("deflate test encoder finishes"); + encoder.into_inner() + } + "br" => { + let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("brotli test body encodes"); + encoder.shutdown().await.expect("brotli test encoder finishes"); + encoder.into_inner() + } + "zstd" => { + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("zstd test body encodes"); + encoder.shutdown().await.expect("zstd test encoder finishes"); + encoder.into_inner() + } + _ => panic!("unsupported test Content-Encoding: {content_encoding}"), + } + } + async fn wait_for_hls_test_session(app_state: &Arc, session_key: &HlsSessionKey) -> HlsSessionHandle { for _ in 0..50 { if let Some(session) = app_state.hls_proxy.sessions().get_by_key(session_key).await { @@ -11604,7 +12024,13 @@ mod tests { response_headers: &'static [(&'static str, &'static str)], body: &'static str, ) -> TestTransientOrigin { - spawn_test_transient_origin_with_delayed_response(status_line, response_headers, body, Duration::ZERO).await + spawn_test_transient_origin_with_delayed_binary_response( + status_line, + response_headers, + body.as_bytes().to_vec(), + Duration::ZERO, + ) + .await } async fn spawn_test_transient_origin_with_delayed_response( @@ -11612,17 +12038,34 @@ mod tests { response_headers: &'static [(&'static str, &'static str)], body: &'static str, response_delay: Duration, + ) -> TestTransientOrigin { + spawn_test_transient_origin_with_delayed_binary_response( + status_line, + response_headers, + body.as_bytes().to_vec(), + response_delay, + ) + .await + } + + async fn spawn_test_transient_origin_with_delayed_binary_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: Vec, + response_delay: Duration, ) -> TestTransientOrigin { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); let addr = listener.local_addr().expect("local addr"); let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); let task_requests = Arc::clone(&requests); + let body = Arc::new(body); let task = tokio::spawn(async move { loop { let Ok((mut socket, _)) = listener.accept().await else { break; }; let requests = Arc::clone(&task_requests); + let body = Arc::clone(&body); tokio::spawn(async move { let mut buf = vec![0_u8; 4096]; let Ok(read) = socket.read(&mut buf).await else { @@ -11636,13 +12079,13 @@ mod tests { if !response_delay.is_zero() { tokio::time::sleep(response_delay).await; } - let mut response = format!("HTTP/1.1 {status_line}\r\nContent-Length: {}\r\n", body.len()); + let mut response_head = format!("HTTP/1.1 {status_line}\r\nContent-Length: {}\r\n", body.len()); for (name, value) in response_headers { - let _ = writeln!(&mut response, "{name}: {value}\r"); + let _ = writeln!(&mut response_head, "{name}: {value}\r"); } - response.push_str("Connection: close\r\n\r\n"); - response.push_str(body); - let _ = socket.write_all(response.as_bytes()).await; + response_head.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response_head.as_bytes()).await; + let _ = socket.write_all(body.as_slice()).await; }); } }); @@ -12110,7 +12553,48 @@ mod tests { assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"456789")); - assert_eq!(origin.requests.lock().await.len(), 1); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); + } + + #[tokio::test] + async fn transient_cache_fill_rejects_identity_partial_without_ready_object_or_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(Arc::clone(&hls_proxy)); + disable_custom_stream_response(&app_state); + let origin = spawn_test_transient_origin_with_response( + "206 Partial Content", + &[("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], + "part", + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let proxy_session_id = ProxySessionId(proxy_session_id); + let cache_key = TransientObjectCacheKey::new(proxy_session_id.clone(), TransientResourceId(resource_id), "ts"); + let session = + hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("session should exist"); + let status = + session.read().await.transient.object_cache.get(&cache_key).expect("transient cache entry").status.clone(); + assert!(matches!(status, TransientObjectCacheStatus::FailedPermanent { status: None, .. })); + assert!(hls_proxy.segment_cache().metadata(&cache_key).await.expect("cache metadata reads").is_none()); + assert!(!hls_proxy.segment_cache().has_active_temp_files().await); + assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); } #[tokio::test] @@ -12284,6 +12768,60 @@ mod tests { wait_for_provider_connection_count(&app_state, 0).await; } + #[tokio::test] + async fn transient_decoder_failure_releases_origin_and_access_guards_once() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let mut truncated = encode_test_manifest("gzip", b"transient decoder failure").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_transient_origin_with_delayed_binary_response( + "200 OK", + &[("Content-Type", "video/mp2t"), ("Content-Encoding", "gzip")], + truncated, + Duration::ZERO, + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id_value) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-a"), + &proxy_session_id_value, + super::current_time_millis(), + )); + } + let resource = session + .read() + .await + .transient + .resources + .get(&TransientResourceId(resource_id.clone())) + .cloned() + .expect("transient resource exists"); + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-")).await; + + assert_eq!(response.status(), StatusCode::OK); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!(resource.active_readers(), 1); + assert!(response.into_body().collect().await.is_err()); + wait_for_provider_connection_count(&app_state, 0).await; + assert_eq!(resource.active_readers(), 0); + tokio::task::yield_now().await; + assert_eq!(app_state.active_provider.get_provider_connections_count().await, 0); + assert_eq!(origin.requests.lock().await.len(), 1, "body failure must not start another origin request"); + } + #[tokio::test] async fn transient_unknown_resource_redirects_to_channel_unavailable_custom_response() { let app_state = test_app_state(); diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 8cd6c714f..f19f5d062 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -11,7 +11,7 @@ use crate::{ endpoints::{ hls_api::{ build_virtual_hls_entry_path, handle_hls_stream_request, hls_admission_failure_manifest_response, - hls_custom_video_manifest_response, m3u_archive_epg_reference_ts, + hls_custom_video_manifest_response, m3u_archive_epg_reference_ts, HlsEntryStreamIdentity, }, xtream_api::{ApiStreamContext, ApiStreamRequest}, }, @@ -305,6 +305,8 @@ pub(in crate::api) async fn m3u_api_stream_loaded( input: &input, user: &user, session_reservation_ttl_secs: get_session_reservation_ttl_secs(app_state, pli.item_type), + content_representation: + crate::api::model::ProviderContentRepresentationMode::for_playback_extension(extension), }, None, ) @@ -396,6 +398,13 @@ pub(in crate::api) async fn m3u_api_stream_loaded( let archive_reference = m3u_archive_epg_reference_ts(&pli.url); // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH if is_session_request && extension == shared::defaults::HLS_EXT { + let Some(stream_identity) = HlsEntryStreamIdentity::from_playlist_item(&pli) else { + error!( + "HLS input stream identity missing for virtual_id={}; refresh target playlist", + pli.virtual_id + ); + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); return handle_hls_stream_request( fingerprint, @@ -405,7 +414,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( user_session.as_ref(), &pli.url, archive_reference, - pli.virtual_id, + stream_identity, &input, req_headers, connection_permission, diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 234351e52..291a8d8b9 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -16,7 +16,7 @@ use crate::{ endpoints::{ hls_api::{ build_virtual_hls_entry_path, handle_hls_stream_request, hls_admission_failure_manifest_response, - hls_custom_video_manifest_response, + hls_custom_video_manifest_response, HlsEntryStreamIdentity, }, xmltv_api::{get_empty_epg_response, get_epg_path_for_target_by_type, serve_short_epg}, }, @@ -483,6 +483,8 @@ async fn xtream_player_api_stream( input: &input, user: &user, session_reservation_ttl_secs: get_session_reservation_ttl_secs(app_state, item_type), + content_representation: + crate::api::model::ProviderContentRepresentationMode::for_playback_extension(playback_ext), }, None, ) @@ -576,6 +578,13 @@ async fn xtream_player_api_stream( let is_session_request = is_session_based_playback(item_type, Some(playback_ext)); // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH if is_session_request && playback_ext == shared::defaults::HLS_EXT { + let Some(stream_identity) = HlsEntryStreamIdentity::from_playlist_item(&pli) else { + error!( + "HLS input stream identity missing for virtual_id={}; refresh target playlist", + pli.virtual_id + ); + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); return handle_hls_stream_request( fingerprint, @@ -585,7 +594,7 @@ async fn xtream_player_api_stream( user_session.as_ref(), &stream_url, None, - pli.virtual_id, + stream_identity, &input, req_headers, connection_permission, @@ -747,6 +756,10 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH if is_session_request && playback_ext == Some(shared::defaults::HLS_EXT) { + let Some(stream_identity) = HlsEntryStreamIdentity::from_playlist_item(&pli) else { + error!("HLS input stream identity missing for virtual_id={virtual_id}; refresh target playlist"); + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); return handle_hls_stream_request( fingerprint, @@ -756,7 +769,7 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( None, &pli.url, None, - virtual_id, + stream_identity, &input, req_headers, UserConnectionPermission::Allowed, @@ -1780,6 +1793,7 @@ mod tests { input_name: "local".intern(), channel_no: 1, source_ordinal: 0, + input_stream_id: "".intern(), } } @@ -1930,6 +1944,7 @@ mod tests { input_name: "strong".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "813563".intern(), } } diff --git a/backend/src/api/model/hls_cache/cache.rs b/backend/src/api/model/hls_cache/cache.rs index 26791852d..f54590860 100644 --- a/backend/src/api/model/hls_cache/cache.rs +++ b/backend/src/api/model/hls_cache/cache.rs @@ -7,13 +7,13 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, RwLock as StdRwLock, }, - time::{Duration, SystemTime}, + time::SystemTime, }; use tokio::{ fs::{self, File, OpenOptions}, io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}, sync::{Mutex, RwLock}, - time::timeout, + time::{timeout_at, Instant}, }; pub const DEFAULT_HLS_CACHE_PATH: &str = "/tmp/tuliprox/cache/hls"; @@ -173,6 +173,31 @@ pub struct StagedCacheObject { pub size: u64, } +/// Typed source used to distinguish a decoded-object size violation from a filesystem failure. +#[derive(Debug, thiserror::Error)] +#[error("hls cache object exceeds configured size limit {limit}")] +pub(crate) struct HlsCacheObjectLimitError { + limit: u64, +} + +impl HlsCacheObjectLimitError { + pub(crate) fn limit(&self) -> u64 { self.limit } +} + +pub(crate) fn hls_cache_object_limit_from_io(error: &io::Error) -> Option<&HlsCacheObjectLimitError> { + let mut source: &(dyn std::error::Error + 'static) = error.get_ref()?; + loop { + if let Some(limit_error) = source.downcast_ref::() { + return Some(limit_error); + } + source = source.source()?; + } +} + +fn cache_object_limit_error(limit: u64) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, HlsCacheObjectLimitError { limit }) +} + #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum CacheInvalidationOutcome { Invalidated, @@ -258,11 +283,11 @@ impl HlsSegmentCache { self.write_temp_and_commit_inner(key, &mut reader, None).await } - pub async fn write_temp_and_commit_with_timeout( + pub async fn write_temp_and_commit_with_deadline( &self, key: &K, mut reader: R, - deadline: Duration, + deadline: Instant, ) -> io::Result where K: HlsCacheObjectKey, @@ -271,11 +296,11 @@ impl HlsSegmentCache { self.write_temp_and_commit_inner(key, &mut reader, Some(deadline)).await } - pub async fn stage_temp_with_timeout( + pub async fn stage_temp_with_deadline( &self, key: &K, mut reader: R, - deadline: Duration, + deadline: Instant, ) -> io::Result where K: HlsCacheObjectKey, @@ -301,8 +326,9 @@ impl HlsSegmentCache { K: HlsCacheObjectKey, { self.ensure_cache_root_marker().await?; - if staged.size > self.max_object_bytes.load(Ordering::Acquire) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "hls cache object exceeds configured size limit")); + let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); + if staged.size > max_object_bytes { + return Err(cache_object_limit_error(max_object_bytes)); } let final_path = self.path_for_key(key); let Some(parent) = final_path.parent() else { @@ -314,12 +340,8 @@ impl HlsSegmentCache { let mut capacity = self.capacity.lock().await; if !capacity.initialized || capacity.cache_path != cache_path { let (total_bytes, session_bytes) = scan_committed_cache_usage(&cache_path).await?; - *capacity = CacheCapacityState { - cache_path: cache_path.clone(), - initialized: true, - total_bytes, - session_bytes, - }; + *capacity = + CacheCapacityState { cache_path: cache_path.clone(), initialized: true, total_bytes, session_bytes }; } let old_size = match fs::metadata(&final_path).await { Ok(metadata) => metadata.len(), @@ -360,7 +382,7 @@ impl HlsSegmentCache { &self, key: &K, reader: &mut R, - deadline: Option, + deadline: Option, ) -> io::Result where K: HlsCacheObjectKey, @@ -374,12 +396,15 @@ impl HlsSegmentCache { &self, key: &K, reader: &mut R, - deadline: Option, + deadline: Option, ) -> io::Result where K: HlsCacheObjectKey, R: AsyncRead + Unpin, { + if deadline.is_some_and(|deadline| deadline <= Instant::now()) { + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } self.ensure_cache_root_marker().await?; let final_path = self.path_for_key(key); let Some(parent) = final_path.parent() else { @@ -388,19 +413,25 @@ impl HlsSegmentCache { fs::create_dir_all(parent).await?; let (temp_path, mut temp_file) = self.create_temp_file(key).await?; + if deadline.is_some_and(|deadline| deadline <= Instant::now()) { + drop(temp_file); + let _ = fs::remove_file(&temp_path).await; + self.unregister_temp_file(&temp_path).await; + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } let copy = async { let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); let mut limited = reader.take(max_object_bytes.saturating_add(1)); let size = tokio::io::copy(&mut limited, &mut temp_file).await?; if size > max_object_bytes { - return Err(io::Error::new(io::ErrorKind::InvalidData, "hls cache object exceeds configured size limit")); + return Err(cache_object_limit_error(max_object_bytes)); } temp_file.flush().await?; drop(temp_file); Ok::(size) }; let copy_result = if let Some(deadline) = deadline { - if let Ok(result) = timeout(deadline, copy).await { + if let Ok(result) = timeout_at(deadline, copy).await { result } else { let _ = fs::remove_file(&temp_path).await; @@ -436,11 +467,7 @@ impl HlsSegmentCache { Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), Err(err) => Err(err), }; - if result.is_ok() - && size > 0 - && capacity.initialized - && capacity.cache_path == self.cache_path_snapshot() - { + if result.is_ok() && size > 0 && capacity.initialized && capacity.cache_path == self.cache_path_snapshot() { capacity.total_bytes = capacity.total_bytes.saturating_sub(size); let session_component = key.session_path_component(); if let Some(session_bytes) = capacity.session_bytes.get_mut(&session_component) { @@ -610,13 +637,7 @@ impl HlsSegmentCache { async fn ensure_cache_root_marker(&self) -> io::Result<()> { let cache_path = self.cache_path_snapshot(); - if self - .marker_path - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - == Some(&cache_path) - { + if self.marker_path.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() == Some(&cache_path) { return Ok(()); } ensure_not_root_like_cache_path(&cache_path)?; @@ -744,9 +765,17 @@ impl Default for HlsSegmentCache { #[cfg(test)] mod tests { - use super::{CacheInvalidationOutcome, HlsSegmentCache, MapCacheKey, SegmentCacheKey, TransientObjectCacheKey}; - use crate::api::model::{build_transient_resource_id, ProxySessionId}; - use std::{collections::HashSet, io, sync::Arc, time::{Duration, SystemTime}}; + use super::{ + hls_cache_object_limit_from_io, CacheInvalidationOutcome, HlsSegmentCache, MapCacheKey, SegmentCacheKey, + TransientObjectCacheKey, + }; + use crate::api::model::{build_transient_resource_id, HlsOriginResourceFetchError, ProxySessionId}; + use std::{ + collections::HashSet, + io, + sync::Arc, + time::{Duration, SystemTime}, + }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn cache_key() -> SegmentCacheKey { SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 123, "ts") } @@ -817,19 +846,35 @@ mod tests { } #[tokio::test] - async fn write_temp_and_commit_with_timeout_cleans_active_temp_file() { + async fn write_temp_and_commit_with_deadline_cleans_active_temp_file() { let temp_dir = tempfile::tempdir().expect("tempdir"); let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); let key = cache_key(); let (_writer, reader) = tokio::io::duplex(64); - let result = cache.write_temp_and_commit_with_timeout(&key, reader, Duration::from_millis(1)).await; + let result = cache + .write_temp_and_commit_with_deadline(&key, reader, tokio::time::Instant::now() + Duration::from_millis(1)) + .await; assert_eq!(result.expect_err("commit should time out").kind(), io::ErrorKind::TimedOut); assert!(!cache.has_active_temp_files().await); assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); } + #[tokio::test] + async fn exhausted_write_deadline_rejects_even_an_immediately_available_body() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + + let result = cache.write_temp_and_commit_with_deadline(&key, &b"ready"[..], tokio::time::Instant::now()).await; + + assert_eq!(result.expect_err("expired deadline must time out").kind(), io::ErrorKind::TimedOut); + assert!(!cache.has_active_temp_files().await); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); + assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); + } + #[tokio::test] async fn cache_object_write_rejects_bytes_above_the_configured_limit() { let temp_dir = tempfile::tempdir().expect("tempdir"); @@ -839,8 +884,15 @@ mod tests { let result = cache.write_bytes_and_commit(&key, b"four").await; - assert!(result.is_err()); + let error = result.expect_err("decoded object above the configured limit must fail"); + let limit_error = hls_cache_object_limit_from_io(&error).expect("typed object-limit source"); + assert_eq!(limit_error.limit(), 3); + assert!(matches!( + HlsOriginResourceFetchError::cache_body(&error), + HlsOriginResourceFetchError::CacheObjectLimit { limit: 3 } + )); assert!(cache.metadata(&key).await.expect("metadata").is_none()); + assert!(!cache.has_active_temp_files().await); } #[tokio::test] @@ -854,10 +906,7 @@ mod tests { tokio::fs::create_dir_all(temp_dir.path().join("orphan")).await.expect("orphan dir"); let cutoff = SystemTime::now(); - let removed = cache - .delete_orphan_session_dirs(&HashSet::from([active]), cutoff) - .await - .expect("cleanup"); + let removed = cache.delete_orphan_session_dirs(&HashSet::from([active]), cutoff).await.expect("cleanup"); assert_eq!(removed, 1); assert!(temp_dir.path().join("active").exists()); @@ -880,10 +929,7 @@ mod tests { filetime::set_file_mtime(temp_dir.path().join(fresh.0.clone()), filetime::FileTime::from_system_time(future)) .expect("set fresh mtime"); - let removed = cache - .delete_orphan_session_dirs(&HashSet::new(), cutoff) - .await - .expect("cleanup"); + let removed = cache.delete_orphan_session_dirs(&HashSet::new(), cutoff).await.expect("cleanup"); assert_eq!(removed, 1, "stale orphan dir should be removed"); assert!(!temp_dir.path().join(stale.0).exists()); diff --git a/backend/src/api/model/hls_cache/gc.rs b/backend/src/api/model/hls_cache/gc.rs index 0e2bad834..ef1fa1db3 100644 --- a/backend/src/api/model/hls_cache/gc.rs +++ b/backend/src/api/model/hls_cache/gc.rs @@ -1,8 +1,7 @@ use super::{ renderer_candidate_window_proxy_seqs, safe_proxy_session_id, CacheInvalidationOutcome, HlsCacheMetrics, - HlsSegmentCache, HlsSession, HlsExpiredSessionReason, HlsSessionHandle, HlsSessionStore, MapCacheKey, - MapCacheStatus, ProxyMapId, - ProxySessionId, SegmentCacheKey, SegmentCacheStatus, TransientObjectCacheKey, + HlsExpiredSessionReason, HlsSegmentCache, HlsSession, HlsSessionHandle, HlsSessionStore, MapCacheKey, + MapCacheStatus, ProxyMapId, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, TransientObjectCacheKey, }; use crate::{api::model::AppState, model::HlsCacheConfig}; use arc_swap::ArcSwap; @@ -203,10 +202,8 @@ impl HlsGarbageCollector { for session in &sessions { active_session_ids.insert(session.read().await.proxy_session_id.clone()); } - report.orphan_session_dirs_deleted = self - .cache - .delete_orphan_session_dirs(&active_session_ids, gc_start) - .await?; + report.orphan_session_dirs_deleted = + self.cache.delete_orphan_session_dirs(&active_session_ids, gc_start).await?; let mut pending_deletions = Vec::new(); for session in &sessions { let mut session = session.write().await; diff --git a/backend/src/api/model/hls_cache/headers.rs b/backend/src/api/model/hls_cache/headers.rs index 1aa7c4c0a..768134b00 100644 --- a/backend/src/api/model/hls_cache/headers.rs +++ b/backend/src/api/model/hls_cache/headers.rs @@ -1,5 +1,8 @@ -use crate::api::model::proxy::header_policy::{HeaderProtocol, HopByHopHeader}; -use crate::model::ReverseProxyDisabledHeaderConfig; +use crate::{ + api::model::proxy::header_policy::{HeaderProtocol, HopByHopHeader}, + model::ReverseProxyDisabledHeaderConfig, + utils::content_coding::force_accept_encoding_identity, +}; use axum::http::{header, HeaderMap, HeaderName, HeaderValue}; use std::collections::HashMap; @@ -25,10 +28,8 @@ pub fn scrub_hls_origin_headers(headers: &mut HeaderMap, disabled_headers: Optio .flat_map(|value| value.split(',')) .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok()) .collect::>(); - names.extend(headers - .keys() - .filter(|name| should_remove_hls_origin_header(name.as_str(), disabled_headers)) - .cloned() + names.extend( + headers.keys().filter(|name| should_remove_hls_origin_header(name.as_str(), disabled_headers)).cloned(), ); for name in names { headers.remove(name); @@ -55,9 +56,8 @@ pub fn extract_hls_provider_session_header_map(headers: &HeaderMap) -> HeaderMap .collect::>(); let mut session_headers = HeaderMap::new(); - if let Some(cookie_header) = (!cookies.is_empty()) - .then(|| cookies.join("; ")) - .and_then(|value| HeaderValue::from_str(&value).ok()) + if let Some(cookie_header) = + (!cookies.is_empty()).then(|| cookies.join("; ")).and_then(|value| HeaderValue::from_str(&value).ok()) { session_headers.insert(header::COOKIE, cookie_header); } @@ -91,7 +91,7 @@ pub fn hls_origin_headers_with_provider_session( pub fn force_identity_without_range(headers: &mut HeaderMap) { headers.remove(header::RANGE); - headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("identity")); + force_accept_encoding_identity(headers); } #[cfg(test)] diff --git a/backend/src/api/model/hls_cache/ids.rs b/backend/src/api/model/hls_cache/ids.rs index 38af01306..49e502ce4 100644 --- a/backend/src/api/model/hls_cache/ids.rs +++ b/backend/src/api/model/hls_cache/ids.rs @@ -5,13 +5,23 @@ const PROXY_SESSION_ID_LEN: usize = 22; const PROXY_SESSION_ID_KEY_CONTEXT: &str = "tuliprox:hls-cache:proxy-session-id-key:v1"; /// Stable Tuliprox content identity for a live HLS source. +/// +/// `stream_ref` is the immutable input-stream ID captured before target mapping. +/// Together with `input_id`, it identifies the origin content across targets; +/// target IDs and virtual IDs must never be used as `stream_ref`. #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub struct HlsSessionKey { + /// Internal ID of the configured Tuliprox input. pub input_id: u16, + /// Exact, non-empty origin/provider stream ID represented as a string. pub stream_ref: String, } impl HlsSessionKey { + /// Creates a key from the configured input ID and its immutable input-stream ID. + /// + /// Callers must pass the origin/provider ID captured before target mapping, not + /// a target-specific or virtual ID. pub fn new(input_id: u16, stream_ref: impl Into) -> Self { Self { input_id, stream_ref: stream_ref.into() } } @@ -56,6 +66,14 @@ mod tests { assert_ne!(first, different_stream); } + #[test] + fn live_hls_session_key_preserves_alphanumeric_input_stream_id() { + let key = HlsSessionKey::new(7, "m3u-channel_A42"); + + assert_eq!(key.stream_ref, "m3u-channel_A42"); + assert_eq!(key.stable_value(), "input:7|hls|m3u-channel_A42"); + } + #[test] fn live_hls_session_key_does_not_contain_origin_or_provider_url_parts() { let key = HlsSessionKey::new(7, "80510"); diff --git a/backend/src/api/model/hls_cache/lease.rs b/backend/src/api/model/hls_cache/lease.rs index aaede6cab..3fa2d6f86 100644 --- a/backend/src/api/model/hls_cache/lease.rs +++ b/backend/src/api/model/hls_cache/lease.rs @@ -98,11 +98,8 @@ impl HlsAccessLeasePendingDeadline { } const fn tightened_with(self, candidate: Self) -> Self { - let deadline_ms = if self.deadline_ms() <= candidate.deadline_ms() { - self.deadline_ms() - } else { - candidate.deadline_ms() - }; + let deadline_ms = + if self.deadline_ms() <= candidate.deadline_ms() { self.deadline_ms() } else { candidate.deadline_ms() }; match (self, candidate) { (Self::FollowUp { .. }, _) | (_, Self::FollowUp { .. }) => Self::FollowUp { deadline_ms }, (Self::Bootstrap { .. }, Self::Bootstrap { .. }) => Self::Bootstrap { deadline_ms }, @@ -302,10 +299,7 @@ impl HlsAccessLeaseStore { .filter(|lease| lease.proxy_session_id == *proxy_session_id) .map(|lease| lease.lease_id.clone()) .collect::>(); - lease_ids - .into_iter() - .filter_map(|lease_id| self.by_lease_id.remove(&lease_id)) - .collect() + lease_ids.into_iter().filter_map(|lease_id| self.by_lease_id.remove(&lease_id)).collect() } pub fn clear(&mut self) -> usize { @@ -325,7 +319,12 @@ impl HlsAccessLeaseStore { .map(|lease| lease.username.clone()) } - pub fn response_snapshot(&mut self, lease_id: &HlsAccessLeaseId, path_proxy_session_id: &ProxySessionId, now_ms: u64) -> Option { + pub fn response_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { let lease = self.by_lease_id.get_mut(lease_id)?; if &lease.proxy_session_id != path_proxy_session_id { return None; @@ -347,10 +346,8 @@ impl HlsAccessLeaseStore { } lease.refresh_validity(now_ms); if lease_state_allows_use(lease.state) { - lease.response_flag = Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { - reason, - set_at_ms: now_ms, - }); + lease.response_flag = + Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { reason, set_at_ms: now_ms }); marked += 1; } } @@ -819,9 +816,7 @@ mod tests { let marked = store.mark_channel_unavailable_for_session( &proxy_session_id, 17_000, - HlsAccessLeaseChannelUnavailableReason::SegmentPermanentFailure { - status: Some(StatusCode::NOT_FOUND), - }, + HlsAccessLeaseChannelUnavailableReason::SegmentPermanentFailure { status: Some(StatusCode::NOT_FOUND) }, ); assert_eq!(marked, 1); @@ -865,11 +860,7 @@ mod tests { set_at_ms: 2_000 }) )); - assert!(store - .response_snapshot(&other_lease_id, &proxy_session_id, 2_000) - .unwrap() - .response_flag - .is_none()); + assert!(store.response_snapshot(&other_lease_id, &proxy_session_id, 2_000).unwrap().response_flag.is_none()); } #[test] @@ -1147,9 +1138,7 @@ mod tests { store.prepare_access_lease(lease(activated_id.clone(), &proxy_session_id.0, 1_000)); store.prepare_access_lease(lease(denied_id.clone(), &proxy_session_id.0, 1_000)); store.prepare_access_lease(lease(expired_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store - .activate_access_lease(&idle_id, &proxy_session_id, 2_000, timing(1_000, 15_000)) - .is_activated()); + assert!(store.activate_access_lease(&idle_id, &proxy_session_id, 2_000, timing(1_000, 15_000)).is_activated()); assert!(store .activate_access_lease(&activated_id, &proxy_session_id, 2_000, timing(5_000, 15_000)) .is_activated()); diff --git a/backend/src/api/model/hls_cache/manager.rs b/backend/src/api/model/hls_cache/manager.rs index 19fbc1f7a..3b4ec3fe8 100644 --- a/backend/src/api/model/hls_cache/manager.rs +++ b/backend/src/api/model/hls_cache/manager.rs @@ -1,19 +1,17 @@ use super::{ - build_rewrite_secret_fingerprint, safe_proxy_session_id, safe_session_key, GarbageCollectionPolicy, - HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, - HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseSessionSnapshot, - HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsCacheMetrics, - HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsGarbageCollector, HlsLifecycleEvent, HlsLifecycleEventKey, - HlsLifecycleManager, HlsMapWorkerPool, - HlsOriginSource, HlsQosRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, - HlsSessionKey, HlsSessionStore, HlsSessionStoreOutcome, ProxySessionId, SegmentFetchPolicy, + build_rewrite_secret_fingerprint, safe_proxy_session_id, safe_session_key, GarbageCollectionPolicy, HlsAccessLease, + HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, + HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseSessionSnapshot, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsCacheMetrics, HlsExpiredSessionMarker, + HlsExpiredSessionReason, HlsGarbageCollector, HlsLifecycleEvent, HlsLifecycleEventKey, HlsLifecycleManager, + HlsMapWorkerPool, HlsOriginSource, HlsQosRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, + HlsSessionHandle, HlsSessionKey, HlsSessionStore, HlsSessionStoreOutcome, ProxySessionId, SegmentFetchPolicy, TransientResourceStore, }; use crate::{ api::model::{ActiveProviderManager, ActiveUserManager, AppState}, - model::{AppConfig, HlsCacheConfig, StripConfig}, + model::{AppConfig, HlsCacheConfig, HlsManifestRecoveryBurstConfig, StripConfig}, }; -use crate::model::HlsManifestRecoveryBurstConfig; use arc_swap::ArcSwap; use log::{debug, error, info}; use shared::utils::sanitize_sensitive_info; @@ -82,10 +80,7 @@ fn hls_pending_manifest_follow_up_window_ms(target_duration: Option) -> u64 target_duration_secs.saturating_mul(2_000).max(10_000) } -fn hls_pending_manifest_follow_up_deadline( - now_ms: u64, - target_duration: Option, -) -> HlsAccessLeasePendingDeadline { +fn hls_pending_manifest_follow_up_deadline(now_ms: u64, target_duration: Option) -> HlsAccessLeasePendingDeadline { HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: now_ms.saturating_add(hls_pending_manifest_follow_up_window_ms(target_duration)), } @@ -219,11 +214,7 @@ impl HlsProxyManager { Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, true) } - fn with_hls_cache_config_and_secret_enabled( - config: &HlsCacheConfig, - rewrite_secret: &[u8], - enabled: bool, - ) -> Self { + fn with_hls_cache_config_and_secret_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { let segment_fetch_policy = SegmentFetchPolicy::from_config(config); let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); let sessions = Arc::new(HlsSessionStore::new()); @@ -320,10 +311,8 @@ impl HlsProxyManager { account_name: &Arc, now_ms: u64, ) -> bool { - let key = HlsAccountOverlapCooldownKey { - input_name: Arc::clone(input_name), - account_name: Arc::clone(account_name), - }; + let key = + HlsAccountOverlapCooldownKey { input_name: Arc::clone(input_name), account_name: Arc::clone(account_name) }; let mut cooldowns = self.account_overlap_cooldowns.write().await; let Some(cooldown) = cooldowns.get(&key).copied() else { return false; @@ -382,10 +371,7 @@ impl HlsProxyManager { return; } let key = HlsAccountOverlapCooldownKey { input_name, account_name }; - self.account_overlap_cooldowns - .write() - .await - .insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); + self.account_overlap_cooldowns.write().await.insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); debug!( "HLS account overlap cooldown set for input {} account {} until {} ms after {}", sanitize_sensitive_info(key.input_name.as_ref()), @@ -402,11 +388,8 @@ impl HlsProxyManager { .reverse_proxy .as_ref() .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); - let hls_config = config - .reverse_proxy - .as_ref() - .and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()) - .cloned(); + let hls_config = + config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()).cloned(); let enabled = hls_config.is_some(); let hls_config = hls_config.unwrap_or_else(|| HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default())); @@ -483,10 +466,7 @@ impl HlsProxyManager { now_ms: u64, reason: HlsAccessLeaseChannelUnavailableReason, ) -> usize { - self.access_leases - .write() - .await - .mark_channel_unavailable_for_session(proxy_session_id, now_ms, reason) + self.access_leases.write().await.mark_channel_unavailable_for_session(proxy_session_id, now_ms, reason) } pub async fn mark_access_lease_channel_unavailable( @@ -560,11 +540,12 @@ impl HlsProxyManager { target_duration: Option, ) -> bool { let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); - let lease = self - .access_leases - .write() - .await - .mark_pending_manifest_follow_up_for_lease(lease_id, proxy_session_id, now_ms, deadline); + let lease = self.access_leases.write().await.mark_pending_manifest_follow_up_for_lease( + lease_id, + proxy_session_id, + now_ms, + deadline, + ); if let Some(lease) = lease { self.schedule_access_lease_validity(&lease).await; debug!( @@ -585,11 +566,11 @@ impl HlsProxyManager { target_duration: Option, ) -> usize { let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); - let leases = self - .access_leases - .write() - .await - .mark_pending_manifest_follow_up_for_session(proxy_session_id, now_ms, deadline); + let leases = self.access_leases.write().await.mark_pending_manifest_follow_up_for_session( + proxy_session_id, + now_ms, + deadline, + ); for lease in &leases { self.schedule_access_lease_validity(lease).await; } @@ -690,11 +671,7 @@ impl HlsProxyManager { now_ms: u64, ) -> Option { self.sessions - .expired_session_marker( - proxy_session_id, - now_ms, - self.session_idle_timeout_ms().saturating_mul(2).max(1), - ) + .expired_session_marker(proxy_session_id, now_ms, self.session_idle_timeout_ms().saturating_mul(2).max(1)) .await } @@ -789,9 +766,7 @@ impl HlsProxyManager { } if snapshot.state != HlsAccessLeaseState::Expired && snapshot.state != HlsAccessLeaseState::Denied { let due_at_ms = if snapshot.state == HlsAccessLeaseState::Pending { - snapshot - .pending_deadline - .map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) + snapshot.pending_deadline.map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) } else { snapshot.valid_until_ms }; @@ -1129,7 +1104,10 @@ mod tests { utils::FileLockManager, }; use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode, ReverseProxyConfigDto}; + use shared::model::{ + ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode, + ReverseProxyConfigDto, + }; use std::sync::Arc; fn empty_paths() -> ConfigPaths { @@ -1207,10 +1185,7 @@ mod tests { assert_eq!(manager.cache_duration_seconds(), 99); assert_eq!(manager.transient_resource_ttl_ms(), 99_000); assert_eq!(manager.session_idle_timeout_ms(), 55_000); - assert_eq!( - manager.manifest_recovery_burst().level, - HlsManifestRecoveryBurstLevel::Balanced - ); + assert_eq!(manager.manifest_recovery_burst().level, HlsManifestRecoveryBurstLevel::Balanced); assert_eq!(manager.strip().mode, HlsStripMode::Seconds); assert_eq!(manager.strip().value, 7); assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); @@ -1222,10 +1197,7 @@ mod tests { assert!(!manager.is_enabled()); assert_eq!( - manager - .run_garbage_collection_once(1_000) - .await - .expect("disabled gc should no-op"), + manager.run_garbage_collection_once(1_000).await.expect("disabled gc should no-op"), super::super::GarbageCollectionReport::default() ); diff --git a/backend/src/api/model/hls_cache/manifest_commit.rs b/backend/src/api/model/hls_cache/manifest_commit.rs index 23026b2aa..9884e2c77 100644 --- a/backend/src/api/model/hls_cache/manifest_commit.rs +++ b/backend/src/api/model/hls_cache/manifest_commit.rs @@ -1,6 +1,6 @@ use super::{ - HlsAccountBindingProtection, HlsFreshManifestRequiredReason, HlsManifestCommitRequirement, - HlsOriginAccountBinding, HlsSession, HlsSessionHandle, HlsSessionMode, HlsSessionStoreOutcome, + HlsAccountBindingProtection, HlsFreshManifestRequiredReason, HlsManifestCommitRequirement, HlsOriginAccountBinding, + HlsSession, HlsSessionHandle, HlsSessionMode, HlsSessionStoreOutcome, }; use std::time::Duration; @@ -78,7 +78,7 @@ pub fn hls_committed_manifest_body_for_request( candidate.rendered_at_ms >= started_at_ms, now_ms, ) && manifest_rendered_after_required_boundary(Some(candidate.rendered_at_ms), options)) - .then_some(candidate.body) + .then_some(candidate.body) }) } @@ -109,9 +109,7 @@ pub async fn hls_manifest_commit_requirement( }; } if matches!(session_outcome, HlsSessionStoreOutcome::Created) { - return HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ColdStart, - }; + return HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; } let session = session.read().await; @@ -181,10 +179,12 @@ fn can_serve_committed_manifest( matches!(policy, HlsCachedManifestPolicy::AllowInitialNoMediaYet) && committed_manifest_valid_at(candidate, now_ms) } - HlsAccountBindingProtection::Expired => matches!(policy, HlsCachedManifestPolicy::AllowInitialNoMediaYet) - && (refreshed_after_wait_started - || (session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_active) - && committed_manifest_valid_at(candidate, now_ms))), + HlsAccountBindingProtection::Expired => { + matches!(policy, HlsCachedManifestPolicy::AllowInitialNoMediaYet) + && (refreshed_after_wait_started + || (session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_active) + && committed_manifest_valid_at(candidate, now_ms))) + } } } diff --git a/backend/src/api/model/hls_cache/manifest_fetch.rs b/backend/src/api/model/hls_cache/manifest_fetch.rs index 7677fa5c1..10a572cb9 100644 --- a/backend/src/api/model/hls_cache/manifest_fetch.rs +++ b/backend/src/api/model/hls_cache/manifest_fetch.rs @@ -1,6 +1,7 @@ use super::{ - extract_hls_provider_session_header_map, safe_origin_log_value, safe_session_key, HlsAccountBindingProtection, - HlsBoundAccountAcquireErrorKind, HlsSessionHandle, HlsSessionMode, TimelineMapError, + extract_hls_provider_session_header_map, log_hls_origin_content_coding, safe_origin_log_value, safe_session_key, + HlsAccountBindingProtection, HlsBoundAccountAcquireErrorKind, HlsOriginContentCodingObjectKind, + HlsOriginContentCodingSource, HlsSessionHandle, HlsSessionMode, TimelineMapError, }; use crate::{ model::{ @@ -11,20 +12,26 @@ use crate::{ parse_manifest_timing, parse_origin_manifest_timeline, parse_origin_media_manifest, OriginManifestParseOutcome, ParsedOriginManifestTimeline, }, - utils::request::{ - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, - send_input_with_retry_and_provider_policy_with_options_result, RequestFetchOptions, + utils::{ + content_coding::{ + apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, + is_http_body_transport_error, read_utf8_limited, ContentBodyReadError, ContentCoding, + ContentCodingDetection, ContentCodingError, OutboundContentCodingPolicy, + }, + request::{ + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_input_with_retry_and_provider_policy_with_options_result, RequestFetchOptions, + }, }, }; use axum::http::{header, HeaderMap, StatusCode}; -use futures::StreamExt; use log::{debug, warn}; use reqwest::Client; use shared::{ - model::{HlsManifestRecoveryBurstLevel, HlsManifestRecoveryBurstPlan, InputFetchMethod, HlsStripMode}, + model::{HlsManifestRecoveryBurstLevel, HlsManifestRecoveryBurstPlan, HlsStripMode, InputFetchMethod}, utils::sanitize_sensitive_info, }; -use std::{collections::HashMap, fmt, future::Future, sync::Arc, time::Duration}; +use std::{collections::HashMap, fmt, future::Future, io, sync::Arc, time::Duration}; use tokio::{task::JoinSet, time::timeout}; use url::Url; @@ -33,7 +40,7 @@ const DEFAULT_HLS_TARGET_DURATION_SECS: u32 = 15; const HLS_MANIFEST_HOST_SWITCH_BASE_WINDOW_SEGMENTS: u32 = 3; const HLS_MANIFEST_HOST_SWITCH_MAX_FAILURE_THRESHOLD: u32 = 5; const DEFAULT_HLS_SESSION_IDLE_TIMEOUT_SECS: u64 = 300; -const MAX_HLS_MANIFEST_BYTES: usize = 2 * 1024 * 1024; +pub(crate) const MAX_HLS_MANIFEST_BYTES: usize = 2 * 1024 * 1024; /// Origin manifest entrypoint snapshot for live HLS refreshes. #[derive(Clone)] @@ -137,16 +144,64 @@ pub fn classify_origin_manifest_status(status: StatusCode) -> OriginManifestStat OriginManifestStatusClass::NonRetryableFailure } -#[derive(Debug)] -pub enum OriginManifestFetchError { +#[derive(Debug, thiserror::Error)] +pub(crate) enum OriginManifestFetchError { + #[error("origin manifest returned permanent status {0}")] PermanentStatus(StatusCode), + #[error("origin manifest returned retryable status {0}, retry_after_ms={1:?}")] RetryableStatus(StatusCode, Option), + #[error("origin manifest retry attempts exhausted")] RetryExhausted, + #[error("origin manifest returned non-retryable status {0}")] NonRetryableStatus(StatusCode), + #[error("origin manifest request failed: {0}")] Request(String), + #[error("origin manifest redirect failed: {0}")] Redirect(String), + #[error("origin manifest request timed out")] Timeout, + #[error("origin provider unavailable: {0:?}")] ProviderUnavailable(HlsBoundAccountAcquireErrorKind), + #[error("origin manifest content coding failed: {0}")] + ContentCoding(ContentCodingError), + #[error("origin manifest decoding failed: coding={coding:?}")] + ContentDecoding { coding: ContentCoding }, + #[error("decoded origin manifest exceeds configured limit {limit}")] + DecodedBodyLimitExceeded { limit: usize }, + #[error("decoded origin manifest is not valid UTF-8: valid_up_to={valid_up_to} error_len={error_len:?}")] + InvalidUtf8 { valid_up_to: usize, error_len: Option }, +} + +impl OriginManifestFetchError { + /// Returns a fixed/numeric diagnostic label without origin-controlled strings. + pub(crate) fn log_label(&self) -> String { + match self { + Self::PermanentStatus(status) => format!("permanent_status status={}", status.as_u16()), + Self::RetryableStatus(status, _) => format!("retryable_status status={}", status.as_u16()), + Self::RetryExhausted => "retry_exhausted".to_string(), + Self::NonRetryableStatus(status) => format!("non_retryable_status status={}", status.as_u16()), + Self::Request(_) => "request".to_string(), + Self::Redirect(_) => "redirect".to_string(), + Self::Timeout => "timeout".to_string(), + Self::ProviderUnavailable(kind) => format!("provider_unavailable kind={}", kind.as_log_label()), + Self::ContentCoding(error) => match error { + ContentCodingError::InvalidHeader => "content_coding class=invalid_header".to_string(), + ContentCodingError::Unsupported(_) => "content_coding class=unsupported".to_string(), + ContentCodingError::EncodedPartialContent => "content_coding class=encoded_partial_content".to_string(), + ContentCodingError::PrefixRead(error) => content_decoding_error_from_io(error).map_or_else( + || "content_coding class=prefix_read".to_string(), + |error| format!("content_decoding coding={}", error.coding.as_http_token()), + ), + }, + Self::ContentDecoding { coding, .. } => { + format!("content_decoding coding={}", coding.as_http_token()) + } + Self::DecodedBodyLimitExceeded { limit } => format!("decoded_body_limit limit={limit}"), + Self::InvalidUtf8 { valid_up_to, error_len } => { + format!("invalid_utf8 valid_up_to={valid_up_to} error_len={error_len:?}") + } + } + } } #[derive(Debug)] @@ -482,40 +537,85 @@ async fn fetch_hls_origin_manifest_initial_global_policy( ); let fetch_options = RequestFetchOptions::with_attempt_idle_timeout(Duration::from_millis( context.origin_manifest_timeout_ms.max(1), - )); - let response_result = if context.use_manual_redirects { - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - &context.app_config, - &context.no_redirect_client, - &input_source, - Some(&context.headers), - context.origin_entry.url(), - MAX_MANUAL_REDIRECTS, - fetch_options, - ) - .await - } else { - send_input_with_retry_and_provider_policy_with_options_result( - &context.app_config, - &context.client, - &input_source, - Some(&context.headers), - context.origin_entry.url(), - fetch_options, - ) - .await - }; - let response_result = response_result.map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; - let provider_url_index = response_result.provider_url_index; - let resolved_request_url = - resolved_hls_manifest_request_url_from_input(&input_source, provider_url_index, context.origin_entry.url()); - response_to_fetched_manifest( - response_result.response, - provider_url_index, - resolved_request_url, - context.origin_manifest_timeout_ms, - ) - .await + )) + .with_content_coding(OutboundContentCodingPolicy::Identity) + .without_resource_retries(); + let attempts = context.retry_policy.attempt_count(); + + // This HLS loop owns the logical manifest-attempt budget. Each iteration may traverse one bounded provider URL + // failover cycle and redirect chain, but the generic resource-retry counter is not nested beneath it. + for attempt_index in 0..attempts { + let response_result = if context.use_manual_redirects { + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &context.app_config, + &context.no_redirect_client, + &input_source, + Some(&context.headers), + context.origin_entry.url(), + MAX_MANUAL_REDIRECTS, + fetch_options, + ) + .await + } else { + send_input_with_retry_and_provider_policy_with_options_result( + &context.app_config, + &context.client, + &input_source, + Some(&context.headers), + context.origin_entry.url(), + fetch_options, + ) + .await + }; + let fetch_result = match response_result { + Ok(response_result) => { + let provider_url_index = response_result.provider_url_index; + let resolved_request_url = resolved_hls_manifest_request_url_from_input( + &input_source, + provider_url_index, + context.origin_entry.url(), + ); + response_to_fetched_manifest( + response_result.response, + provider_url_index, + resolved_request_url, + context.origin_manifest_timeout_ms, + ) + .await + } + Err(err) => Err(origin_manifest_fetch_error_from_io_error(&err)), + }; + match fetch_result { + Ok(fetched) => return Ok(fetched.with_attempts(attempt_index + 1)), + Err(err) if is_hls_retryable_initial_manifest_fetch_error(&err) && attempt_index + 1 < attempts => { + let retry_after_ms = match &err { + OriginManifestFetchError::RetryableStatus(_, retry_after_ms) => *retry_after_ms, + _ => None, + }; + let jitter_ms = if retry_after_ms.is_some() || context.retry_policy.jitter_max_ms == 0 { + 0 + } else { + fastrand::u64(0..=context.retry_policy.jitter_max_ms) + }; + let delay_ms = next_retry_delay_ms(&context.retry_policy, attempt_index, retry_after_ms, jitter_ms); + log_manifest_retry_scheduled( + context, + ManifestRetryLogKind::InitialFetch, + attempt_index, + attempts, + delay_ms, + None, + Some(&err), + ) + .await; + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + } + Err(err) => return Err(err), + } + } + Err(OriginManifestFetchError::RetryExhausted) } pub(crate) async fn score_hls_manifest_candidate_for_selection_log( @@ -578,9 +678,10 @@ where HlsManifestRecoveryAttemptError::Rejected(reason) if attempt_index + 1 < attempts => { log_manifest_retry_scheduled( context, + ManifestRetryLogKind::PinnedHostRecovery, attempt_index, attempts, - next_retry_delay_ms(&context.retry_policy, attempt_index, None), + next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), Some(&reason), None, ) @@ -596,9 +697,10 @@ where { log_manifest_retry_scheduled( context, + ManifestRetryLogKind::PinnedHostRecovery, attempt_index, attempts, - next_retry_delay_ms(&context.retry_policy, attempt_index, None), + next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), None, Some(&err), ) @@ -1301,17 +1403,36 @@ pub(crate) fn fetched_effective_manifest_host(fetched: &FetchedOriginManifest) - Url::parse(&fetched.resolved_request_url).ok().and_then(|url| url.host_str().map(str::to_string)) } -fn is_hls_retryable_manifest_reject_fetch_error(err: &OriginManifestFetchError) -> bool { +fn is_hls_retryable_initial_manifest_fetch_error(err: &OriginManifestFetchError) -> bool { matches!( err, OriginManifestFetchError::RetryableStatus(_, _) | OriginManifestFetchError::Request(_) | OriginManifestFetchError::Redirect(_) | OriginManifestFetchError::Timeout - | OriginManifestFetchError::RetryExhausted + | OriginManifestFetchError::ContentDecoding { .. } + | OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(_)) ) } +fn is_hls_retryable_manifest_reject_fetch_error(err: &OriginManifestFetchError) -> bool { + match err { + OriginManifestFetchError::RetryableStatus(_, _) + | OriginManifestFetchError::Request(_) + | OriginManifestFetchError::Redirect(_) + | OriginManifestFetchError::Timeout + | OriginManifestFetchError::RetryExhausted + | OriginManifestFetchError::ContentDecoding { .. } + | OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(_)) => true, + OriginManifestFetchError::PermanentStatus(_) + | OriginManifestFetchError::NonRetryableStatus(_) + | OriginManifestFetchError::ProviderUnavailable(_) + | OriginManifestFetchError::ContentCoding(_) + | OriginManifestFetchError::DecodedBodyLimitExceeded { .. } + | OriginManifestFetchError::InvalidUtf8 { .. } => false, + } +} + pub(crate) fn commit_error_to_fetch_error(err: &HlsManifestCommitError) -> OriginManifestFetchError { match err { HlsManifestCommitError::TimelineRejected { .. } | HlsManifestCommitError::RetryCurrentTarget => { @@ -1347,12 +1468,25 @@ pub(crate) fn next_committed_origin_highwater( } } -fn next_retry_delay_ms(retry_policy: &RetryPolicy, attempt_index: usize, retry_after_ms: Option) -> u64 { - retry_after_ms.unwrap_or_else(|| retry_policy.delays_ms.get(attempt_index + 1).copied().unwrap_or_default()) +fn next_retry_delay_ms( + retry_policy: &RetryPolicy, + attempt_index: usize, + retry_after_ms: Option, + jitter_ms: u64, +) -> u64 { + retry_after_ms + .unwrap_or_else(|| retry_policy.delay_for_attempt_ms(attempt_index + 1, jitter_ms).unwrap_or_default()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ManifestRetryLogKind { + InitialFetch, + PinnedHostRecovery, } async fn log_manifest_retry_scheduled( context: &HlsOriginManifestFetchContext, + retry_kind: ManifestRetryLogKind, attempt_index: usize, attempts: usize, delay_ms: u64, @@ -1363,7 +1497,7 @@ async fn log_manifest_retry_scheduled( let session = context.session.read().await; safe_session_key(&session.key) }; - let status = manifest_retry_status_label(reject_reason, fetch_error); + let status = manifest_retry_status_label(retry_kind, reject_reason, fetch_error); warn!( "Manifest '{}' retry scheduled: status {} attempt {} of {} next_delay_ms={delay_ms}", session_label, @@ -1374,16 +1508,21 @@ async fn log_manifest_retry_scheduled( } fn manifest_retry_status_label( + retry_kind: ManifestRetryLogKind, reject_reason: Option<&HlsManifestRejectLogReason>, fetch_error: Option<&OriginManifestFetchError>, ) -> String { - match (reject_reason, fetch_error) { - (Some(reason), Some(err)) => { - format!("{} error={}", reason.status_label(), safe_origin_log_value(format!("{err:?}"))) + match (retry_kind, reject_reason, fetch_error) { + (_, Some(reason), Some(err)) => format!("{} error={}", reason.status_label(), err.log_label()), + (_, Some(reason), None) => reason.status_label(), + (ManifestRetryLogKind::InitialFetch, None, Some(err)) => { + format!("initial-fetch error={}", err.log_label()) } - (Some(reason), None) => reason.status_label(), - (None, Some(err)) => format!("pinned-host-recovery error={}", safe_origin_log_value(format!("{err:?}"))), - (None, None) => "pinned-host-recovery".to_string(), + (ManifestRetryLogKind::InitialFetch, None, None) => "initial-fetch".to_string(), + (ManifestRetryLogKind::PinnedHostRecovery, None, Some(err)) => { + format!("pinned-host-recovery error={}", err.log_label()) + } + (ManifestRetryLogKind::PinnedHostRecovery, None, None) => "pinned-host-recovery".to_string(), } } @@ -1406,12 +1545,14 @@ async fn fetch_origin_manifest_once( ) .await } else { + let mut request_headers = headers.clone(); + apply_outbound_content_coding_policy(&mut request_headers, OutboundContentCodingPolicy::Identity); let response = client .get(entry_url.clone()) - .headers(headers.clone()) + .headers(request_headers) .send() .await - .map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + .map_err(|err| origin_manifest_fetch_error_from_reqwest_error(&err))?; response_to_fetched_manifest(response, provider_url_index, entry_url.clone(), origin_manifest_timeout_ms).await } } @@ -1428,12 +1569,14 @@ async fn fetch_origin_manifest_with_manual_redirects( let mut remaining_redirects = MAX_MANUAL_REDIRECTS; loop { + let mut request_headers = current_headers.clone(); + apply_outbound_content_coding_policy(&mut request_headers, OutboundContentCodingPolicy::Identity); let response = client .get(current_url.clone()) - .headers(current_headers.clone()) + .headers(request_headers) .send() .await - .map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + .map_err(|err| origin_manifest_fetch_error_from_reqwest_error(&err))?; if !response.status().is_redirection() { return response_to_fetched_manifest( response, @@ -1544,18 +1687,17 @@ async fn response_to_fetched_manifest( ); match classify_origin_manifest_status(status) { OriginManifestStatusClass::Success => { - let final_url = response.url().clone(); - let redirect_host = hls_manifest_redirect_host(&resolved_request_url, &final_url); - let provider_session_headers = extract_hls_provider_session_header_map(response.headers()); - let body = read_origin_manifest_body(response, origin_manifest_timeout_ms).await?; + let (decoded, body) = read_origin_manifest_body(response, origin_manifest_timeout_ms).await?; + let redirect_host = hls_manifest_redirect_host(&resolved_request_url, &decoded.final_url); + let provider_session_headers = extract_hls_provider_session_header_map(&decoded.headers); Ok(FetchedOriginManifest { body, - final_manifest_url: final_url.to_string(), + final_manifest_url: decoded.final_url.to_string(), resolved_request_url: resolved_request_url.to_string(), redirect_host, provider_url_index, provider_session_headers, - status, + status: decoded.status, attempts: 1, }) } @@ -1567,37 +1709,66 @@ async fn response_to_fetched_manifest( } } -fn manifest_body_size_after_chunk(current: usize, chunk: usize) -> Option { - current.checked_add(chunk).filter(|size| *size <= MAX_HLS_MANIFEST_BYTES) -} - async fn read_origin_manifest_body( response: reqwest::Response, origin_manifest_timeout_ms: u64, -) -> Result { - if response.content_length().is_some_and(|size| size > MAX_HLS_MANIFEST_BYTES as u64) { - return Err(OriginManifestFetchError::NonRetryableStatus(StatusCode::PAYLOAD_TOO_LARGE)); - } +) -> Result<(crate::utils::content_coding::DecodedHttpResponse, String), OriginManifestFetchError> { let read = async move { - let capacity = response.content_length().map_or(0, |size| { - usize::try_from(size.min(MAX_HLS_MANIFEST_BYTES as u64)).unwrap_or(MAX_HLS_MANIFEST_BYTES) - }); - let mut body = Vec::with_capacity(capacity); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; - if manifest_body_size_after_chunk(body.len(), chunk.len()).is_none() { - return Err(OriginManifestFetchError::NonRetryableStatus(StatusCode::PAYLOAD_TOO_LARGE)); - } - body.extend_from_slice(&chunk); + let mut decoded = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrKnownHlsManifestMagic) + .await + .map_err(origin_manifest_content_coding_error)?; + if let Some(observation) = decoded.content_coding_observation() { + log_hls_origin_content_coding( + observation, + HlsOriginContentCodingObjectKind::Manifest, + false, + HlsOriginContentCodingSource::Shared, + ); } - String::from_utf8(body).map_err(|_| OriginManifestFetchError::Request("origin manifest is not UTF-8".to_string())) + let body = read_utf8_limited(&mut decoded.body, MAX_HLS_MANIFEST_BYTES) + .await + .map_err(origin_manifest_body_read_error)?; + Ok((decoded, body)) }; timeout(Duration::from_millis(origin_manifest_timeout_ms.max(1)), read) .await .map_err(|_| OriginManifestFetchError::Timeout)? } +fn origin_manifest_content_coding_error(error: ContentCodingError) -> OriginManifestFetchError { + match error { + ContentCodingError::PrefixRead(io_error) => { + if let Some(decoding_error) = content_decoding_error_from_io(&io_error) { + OriginManifestFetchError::ContentDecoding { coding: decoding_error.coding } + } else if io_error.kind() == io::ErrorKind::TimedOut { + OriginManifestFetchError::Timeout + } else if is_http_body_transport_error(&io_error) { + origin_manifest_fetch_error_from_io_error(&io_error) + } else { + OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(io_error)) + } + } + error => OriginManifestFetchError::ContentCoding(error), + } +} + +fn origin_manifest_body_read_error(error: ContentBodyReadError) -> OriginManifestFetchError { + match error { + ContentBodyReadError::LimitExceeded { limit } => OriginManifestFetchError::DecodedBodyLimitExceeded { limit }, + ContentBodyReadError::InvalidUtf8 { valid_up_to, error_len } => { + OriginManifestFetchError::InvalidUtf8 { valid_up_to, error_len } + } + ContentBodyReadError::Io(io_error) => { + if let Some(decoding_error) = content_decoding_error_from_io(&io_error) { + OriginManifestFetchError::ContentDecoding { coding: decoding_error.coding } + } else { + origin_manifest_fetch_error_from_io_error(&io_error) + } + } + } +} + pub(crate) fn hls_manifest_redirect_host(resolved_request_url: &Url, final_url: &Url) -> Option { let final_host = final_url.host_str()?; (resolved_request_url.host_str() != Some(final_host)).then(|| final_host.to_string()) @@ -1649,6 +1820,22 @@ fn origin_manifest_fetch_error_from_request_error(err: &impl ToString) -> Origin } } +fn origin_manifest_fetch_error_from_reqwest_error(err: &reqwest::Error) -> OriginManifestFetchError { + if err.is_timeout() { + OriginManifestFetchError::Timeout + } else { + origin_manifest_fetch_error_from_request_error(err) + } +} + +fn origin_manifest_fetch_error_from_io_error(err: &io::Error) -> OriginManifestFetchError { + if err.kind() == io::ErrorKind::TimedOut { + OriginManifestFetchError::Timeout + } else { + origin_manifest_fetch_error_from_request_error(err) + } +} + fn request_failed_status_from_message(message: &str) -> Option { let marker = "Request failed ("; let status_start = message.find(marker)?.checked_add(marker.len())?; @@ -1730,7 +1917,7 @@ pub(crate) async fn refresh_from_live_hls_entrypoint_with_retries( log_origin_refresh_retry_scheduled( origin_entry, attempt_index, - next_retry_delay_ms(retry_policy, attempt_index, retry_after_ms), + next_retry_delay_ms(retry_policy, attempt_index, retry_after_ms, 0), format!("status={}", status.as_u16()), ); retry_after_delay_ms = retry_after_ms; @@ -1742,8 +1929,22 @@ pub(crate) async fn refresh_from_live_hls_entrypoint_with_retries( log_origin_refresh_retry_scheduled( origin_entry, attempt_index, - next_retry_delay_ms(retry_policy, attempt_index, None), - format!("error={}", safe_origin_log_value(&err)), + next_retry_delay_ms(retry_policy, attempt_index, None, 0), + "error=request".to_string(), + ); + } + Ok(Err( + err @ (OriginManifestFetchError::ContentDecoding { .. } + | OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(_))), + )) => { + if attempt_index + 1 == attempts { + return Err(err); + } + log_origin_refresh_retry_scheduled( + origin_entry, + attempt_index, + next_retry_delay_ms(retry_policy, attempt_index, None, 0), + format!("error={}", err.log_label()), ); } Ok(Err(err @ (OriginManifestFetchError::Redirect(_) | OriginManifestFetchError::Timeout))) => { @@ -1753,14 +1954,19 @@ pub(crate) async fn refresh_from_live_hls_entrypoint_with_retries( log_origin_refresh_retry_scheduled( origin_entry, attempt_index, - next_retry_delay_ms(retry_policy, attempt_index, None), - format!("error={}", safe_origin_log_value(format!("{err:?}"))), + next_retry_delay_ms(retry_policy, attempt_index, None, 0), + format!("error={}", err.log_label()), ); } Ok(Err(OriginManifestFetchError::RetryExhausted)) => return Err(OriginManifestFetchError::RetryExhausted), Ok(Err(OriginManifestFetchError::ProviderUnavailable(kind))) => { return Err(OriginManifestFetchError::ProviderUnavailable(kind)); } + Ok(Err( + err @ (OriginManifestFetchError::ContentCoding(_) + | OriginManifestFetchError::DecodedBodyLimitExceeded { .. } + | OriginManifestFetchError::InvalidUtf8 { .. }), + )) => return Err(err), Err(OriginManifestFetchError::Timeout) => { if attempt_index + 1 == attempts { return Err(OriginManifestFetchError::Timeout); @@ -1768,7 +1974,7 @@ pub(crate) async fn refresh_from_live_hls_entrypoint_with_retries( log_origin_refresh_retry_scheduled( origin_entry, attempt_index, - next_retry_delay_ms(retry_policy, attempt_index, None), + next_retry_delay_ms(retry_policy, attempt_index, None, 0), "error=timeout", ); } @@ -1797,10 +2003,130 @@ fn log_origin_refresh_retry_scheduled( #[cfg(test)] mod tests { use super::{ - manifest_body_size_after_chunk, origin_manifest_fetch_error_from_request_error, request_failed_status_from_message, - OriginManifestFetchError, MAX_HLS_MANIFEST_BYTES, + fetch_origin_manifest_once, origin_manifest_content_coding_error, origin_manifest_fetch_error_from_io_error, + origin_manifest_fetch_error_from_request_error, request_failed_status_from_message, ManifestRetryLogKind, + OriginManifestFetchError, RetryPolicy, MAX_HLS_MANIFEST_BYTES, }; - use axum::http::StatusCode; + use crate::utils::content_coding::{ContentCoding, ContentCodingError}; + use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; + use flate2::{ + write::{DeflateEncoder, GzEncoder}, + Compression, + }; + use reqwest::{redirect::Policy, Client}; + use std::{io, io::Write, time::Duration}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::oneshot, + }; + use url::Url; + + const TEST_MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXTINF:2,\nsegment.ts\n"; + + struct TestOriginResponse { + status: &'static str, + headers: Vec<(&'static str, String)>, + body: Vec, + body_delay: Duration, + } + + impl TestOriginResponse { + fn ok(body: Vec) -> Self { + Self { status: "200 OK", headers: Vec::new(), body, body_delay: Duration::ZERO } + } + + fn with_header(mut self, name: &'static str, value: impl Into) -> Self { + self.headers.push((name, value.into())); + self + } + } + + async fn spawn_test_origin(response: TestOriginResponse) -> (Url, oneshot::Receiver) { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.expect("bind test origin"); + let address = listener.local_addr().expect("test origin address"); + let (request_sender, request_receiver) = oneshot::channel(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept test request"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buffer).await.expect("read test request"); + if count == 0 { + break; + } + request.extend_from_slice(&buffer[..count]); + } + let _ = request_sender.send(String::from_utf8_lossy(&request).into_owned()); + + let mut response_head = format!( + "HTTP/1.1 {}\r\nConnection: close\r\nContent-Length: {}\r\n", + response.status, + response.body.len() + ); + for (name, value) in response.headers { + response_head.push_str(name); + response_head.push_str(": "); + response_head.push_str(&value); + response_head.push_str("\r\n"); + } + response_head.push_str("\r\n"); + socket.write_all(response_head.as_bytes()).await.expect("write test response headers"); + if !response.body_delay.is_zero() { + tokio::time::sleep(response.body_delay).await; + } + let _ = socket.write_all(&response.body).await; + }); + (Url::parse(&format!("http://{address}/manifest.m3u8")).expect("test origin URL"), request_receiver) + } + + fn gzip(input: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(input).expect("gzip input"); + encoder.finish().expect("finish gzip") + } + + fn raw_deflate(input: &[u8]) -> Vec { + let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(input).expect("raw-deflate input"); + encoder.finish().expect("finish raw-deflate") + } + + fn brotli(input: &[u8]) -> Vec { + let mut output = Vec::new(); + { + let mut encoder = brotli::CompressorWriter::new(&mut output, 4096, 5, 22); + encoder.write_all(input).expect("brotli input"); + } + output + } + + async fn zstd(input: &[u8]) -> Vec { + let (writer, mut reader) = tokio::io::duplex(64 * 1024); + let input = input.to_vec(); + let encoder_task = tokio::spawn(async move { + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(writer); + encoder.write_all(&input).await.expect("zstd input"); + encoder.shutdown().await.expect("finish zstd"); + }); + let mut output = Vec::new(); + reader.read_to_end(&mut output).await.expect("read zstd output"); + encoder_task.await.expect("join zstd encoder"); + output + } + + fn captured_header<'a>(request: &'a str, name: &str) -> Option<&'a str> { + request.lines().skip(1).find_map(|line| { + let (header_name, value) = line.split_once(':')?; + header_name.eq_ignore_ascii_case(name).then(|| value.trim()) + }) + } + + fn test_clients() -> (Client, Client) { + let client = Client::builder().build().expect("test client"); + let no_redirect_client = Client::builder().redirect(Policy::none()).build().expect("no-redirect client"); + (client, no_redirect_client) + } #[test] fn request_failed_status_is_extracted_from_global_provider_policy_error() { @@ -1812,10 +2138,43 @@ mod tests { ); } + #[test] + fn initial_manifest_retry_delay_uses_next_slot_bounded_jitter_and_retry_after_override() { + let retry_policy = RetryPolicy { delays_ms: [0, 100, 250, 500, 750], jitter_max_ms: 25 }; + + assert_eq!(super::next_retry_delay_ms(&retry_policy, 0, None, 17), 117); + assert_eq!(super::next_retry_delay_ms(&retry_policy, 0, None, 100), 125); + assert_eq!(super::next_retry_delay_ms(&retry_policy, 1, None, 10), 260); + assert_eq!(super::next_retry_delay_ms(&retry_policy, 0, Some(3_000), 25), 3_000); + } + + #[test] + fn initial_manifest_retry_status_is_not_labeled_as_recovery() { + let error = OriginManifestFetchError::Timeout; + + let initial = super::manifest_retry_status_label(ManifestRetryLogKind::InitialFetch, None, Some(&error)); + let recovery = super::manifest_retry_status_label(ManifestRetryLogKind::PinnedHostRecovery, None, Some(&error)); + + assert!(initial.starts_with("initial-fetch error=")); + assert!(!initial.contains("pinned-host-recovery")); + assert!(recovery.starts_with("pinned-host-recovery error=")); + } + + #[test] + fn manifest_content_coding_log_labels_never_include_origin_controlled_details() { + let unsupported = + OriginManifestFetchError::ContentCoding(ContentCodingError::Unsupported("signed-token-secret".to_string())); + assert_eq!(unsupported.log_label(), "content_coding class=unsupported"); + assert!(!unsupported.log_label().contains("signed-token-secret")); + + let decoding = OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Brotli }; + assert_eq!(decoding.log_label(), "content_decoding coding=br"); + } + #[test] fn request_failed_407_maps_to_retryable_manifest_status() { - let err = origin_manifest_fetch_error_from_request_error(& - "Request failed (407 Proxy Authentication Required): provider://demo/live/u/p/1.m3u8", + let err = origin_manifest_fetch_error_from_request_error( + &"Request failed (407 Proxy Authentication Required): provider://demo/live/u/p/1.m3u8", ); assert!(matches!( err, @@ -1839,20 +2198,211 @@ mod tests { #[test] fn request_failed_404_maps_to_permanent_manifest_status() { - let err = - origin_manifest_fetch_error_from_request_error(&"Request failed (404 Not Found): http://example.test/live.m3u8"); + let err = origin_manifest_fetch_error_from_request_error( + &"Request failed (404 Not Found): http://example.test/live.m3u8", + ); assert!(matches!(err, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); } #[test] fn transport_error_without_http_status_stays_request_error() { let err = origin_manifest_fetch_error_from_request_error(&"error sending request for url"); - assert!(matches!(err, OriginManifestFetchError::Request(message) if message == "error sending request for url")); + assert!( + matches!(err, OriginManifestFetchError::Request(message) if message == "error sending request for url") + ); } #[test] - fn manifest_body_limit_rejects_the_first_byte_above_the_limit() { - assert_eq!(manifest_body_size_after_chunk(MAX_HLS_MANIFEST_BYTES - 1, 1), Some(MAX_HLS_MANIFEST_BYTES)); - assert_eq!(manifest_body_size_after_chunk(MAX_HLS_MANIFEST_BYTES, 1), None); + fn io_timeout_maps_to_structured_manifest_timeout() { + let error = io::Error::new(io::ErrorKind::TimedOut, "origin body timed out"); + assert!(matches!(origin_manifest_fetch_error_from_io_error(&error), OriginManifestFetchError::Timeout)); + let prefix_timeout = + ContentCodingError::PrefixRead(io::Error::new(io::ErrorKind::TimedOut, "origin prefix timed out")); + assert!(matches!(origin_manifest_content_coding_error(prefix_timeout), OriginManifestFetchError::Timeout)); + } + + #[tokio::test] + async fn shared_manifest_decodes_supported_origin_content_codings_and_keeps_provider_cookie() { + let encoded_bodies = [ + ("gzip", gzip(TEST_MANIFEST)), + ("deflate", raw_deflate(TEST_MANIFEST)), + ("br", brotli(TEST_MANIFEST)), + ("zstd", zstd(TEST_MANIFEST).await), + ]; + + for (coding, encoded_body) in encoded_bodies { + let response = TestOriginResponse::ok(encoded_body) + .with_header("Content-Encoding", coding) + .with_header("Set-Cookie", format!("sid={coding}; Path=/")); + let (entry_url, request_receiver) = spawn_test_origin(response).await; + let (client, no_redirect_client) = test_clients(); + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + + let fetched = + fetch_origin_manifest_once(&entry_url, &headers, &client, &no_redirect_client, false, None, 1_000) + .await + .unwrap_or_else(|error| panic!("decode {coding} manifest: {error:?}")); + + assert_eq!(fetched.body.as_bytes(), TEST_MANIFEST, "coding={coding}"); + assert_eq!( + fetched + .provider_session_headers + .get(header::COOKIE) + .expect("provider cookie") + .to_str() + .expect("provider cookie text"), + format!("sid={coding}"), + "coding={coding}" + ); + let request = request_receiver.await.expect("captured request"); + assert_eq!(captured_header(&request, "accept-encoding"), Some("identity"), "coding={coding}"); + } + } + + #[tokio::test] + async fn shared_manifest_magic_sniffs_gzip_only_in_manifest_mode() { + let (entry_url, _) = spawn_test_origin(TestOriginResponse::ok(gzip(TEST_MANIFEST))).await; + let (client, no_redirect_client) = test_clients(); + + let fetched = + fetch_origin_manifest_once(&entry_url, &HeaderMap::new(), &client, &no_redirect_client, false, None, 1_000) + .await + .expect("decode magic-sniffed gzip manifest"); + + assert_eq!(fetched.body.as_bytes(), TEST_MANIFEST); + } + + #[tokio::test] + async fn direct_manual_redirect_reapplies_identity_after_cross_origin_scrubbing() { + let (target_url, target_request_receiver) = + spawn_test_origin(TestOriginResponse::ok(TEST_MANIFEST.to_vec())).await; + let redirect_response = TestOriginResponse { + status: "302 Found", + headers: vec![("Location", target_url.to_string())], + body: Vec::new(), + body_delay: Duration::ZERO, + }; + let (entry_url, redirect_request_receiver) = spawn_test_origin(redirect_response).await; + let (client, no_redirect_client) = test_clients(); + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + + let fetched = fetch_origin_manifest_once(&entry_url, &headers, &client, &no_redirect_client, true, None, 1_000) + .await + .expect("follow manual manifest redirect"); + assert_eq!(fetched.body.as_bytes(), TEST_MANIFEST); + + let redirect_request = redirect_request_receiver.await.expect("captured redirect request"); + let target_request = target_request_receiver.await.expect("captured target request"); + assert_eq!(captured_header(&redirect_request, "accept-encoding"), Some("identity")); + assert_eq!(captured_header(&target_request, "accept-encoding"), Some("identity")); + assert!(captured_header(&target_request, "authorization").is_none()); + assert!(captured_header(&target_request, "cookie").is_none()); + } + + #[tokio::test] + async fn shared_manifest_limit_applies_after_decompression() { + let decoded_body = vec![b'x'; MAX_HLS_MANIFEST_BYTES + 1]; + let response = TestOriginResponse::ok(gzip(&decoded_body)).with_header("Content-Encoding", "gzip"); + let (entry_url, _) = spawn_test_origin(response).await; + let (client, no_redirect_client) = test_clients(); + + let error = + fetch_origin_manifest_once(&entry_url, &HeaderMap::new(), &client, &no_redirect_client, false, None, 2_000) + .await + .expect_err("decoded manifest above limit must fail"); + + assert!(matches!( + error, + OriginManifestFetchError::DecodedBodyLimitExceeded { limit } if limit == MAX_HLS_MANIFEST_BYTES + )); + } + + #[tokio::test] + async fn shared_manifest_deadline_includes_magic_prefix_read() { + let response = TestOriginResponse { + body_delay: Duration::from_millis(100), + ..TestOriginResponse::ok(TEST_MANIFEST.to_vec()) + }; + let (entry_url, _) = spawn_test_origin(response).await; + let (client, no_redirect_client) = test_clients(); + + let error = + fetch_origin_manifest_once(&entry_url, &HeaderMap::new(), &client, &no_redirect_client, false, None, 10) + .await + .expect_err("prefix read must honor manifest deadline"); + + assert!(matches!(error, OriginManifestFetchError::Timeout)); + } + + #[tokio::test] + async fn shared_manifest_encoded_body_timeout_stays_structured_timeout() { + let response = TestOriginResponse { + body_delay: Duration::from_millis(100), + ..TestOriginResponse::ok(gzip(TEST_MANIFEST)).with_header("Content-Encoding", "gzip") + }; + let (entry_url, _) = spawn_test_origin(response).await; + let (client, no_redirect_client) = test_clients(); + + let error = + fetch_origin_manifest_once(&entry_url, &HeaderMap::new(), &client, &no_redirect_client, false, None, 10) + .await + .expect_err("encoded body read must honor manifest deadline"); + + assert!(matches!(error, OriginManifestFetchError::Timeout)); + } + + #[tokio::test] + async fn shared_manifest_distinguishes_invalid_utf8_from_decoder_failure() { + let (invalid_utf8_url, _) = spawn_test_origin(TestOriginResponse::ok(vec![0xff])).await; + let corrupt_gzip = TestOriginResponse::ok(vec![0x1f, 0x8b, 0x08, 0x00]).with_header("Content-Encoding", "gzip"); + let (corrupt_gzip_url, _) = spawn_test_origin(corrupt_gzip).await; + let (client, no_redirect_client) = test_clients(); + + let invalid_utf8 = fetch_origin_manifest_once( + &invalid_utf8_url, + &HeaderMap::new(), + &client, + &no_redirect_client, + false, + None, + 1_000, + ) + .await + .expect_err("invalid UTF-8 must fail"); + let decoder_failure = fetch_origin_manifest_once( + &corrupt_gzip_url, + &HeaderMap::new(), + &client, + &no_redirect_client, + false, + None, + 1_000, + ) + .await + .expect_err("corrupt gzip must fail"); + + assert!(matches!(invalid_utf8, OriginManifestFetchError::InvalidUtf8 { .. })); + assert!(matches!(decoder_failure, OriginManifestFetchError::ContentDecoding { .. })); + } + + #[tokio::test] + async fn shared_manifest_rejects_encoded_partial_content() { + let response = TestOriginResponse { + status: "206 Partial Content", + ..TestOriginResponse::ok(gzip(TEST_MANIFEST)).with_header("Content-Encoding", "gzip") + }; + let (entry_url, _) = spawn_test_origin(response).await; + let (client, no_redirect_client) = test_clients(); + + let error = + fetch_origin_manifest_once(&entry_url, &HeaderMap::new(), &client, &no_redirect_client, false, None, 1_000) + .await + .expect_err("encoded partial manifest must fail"); + + assert!(matches!(error, OriginManifestFetchError::ContentCoding(ContentCodingError::EncodedPartialContent))); } } diff --git a/backend/src/api/model/hls_cache/map.rs b/backend/src/api/model/hls_cache/map.rs index 4a4be70e3..ec758fafd 100644 --- a/backend/src/api/model/hls_cache/map.rs +++ b/backend/src/api/model/hls_cache/map.rs @@ -1,6 +1,6 @@ use super::{cache::MapCacheKey, ids::ProxySessionId, session::HlsSession, timeline::CacheAccessState}; -use axum::http::StatusCode; use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use axum::http::StatusCode; use std::{ fmt, hash::{Hash, Hasher}, diff --git a/backend/src/api/model/hls_cache/map_fetcher.rs b/backend/src/api/model/hls_cache/map_fetcher.rs index 7497f36dd..8a5658c74 100644 --- a/backend/src/api/model/hls_cache/map_fetcher.rs +++ b/backend/src/api/model/hls_cache/map_fetcher.rs @@ -2,20 +2,19 @@ use super::{ begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, finish_hls_origin_account_io, hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseStore, HlsBoundAccountAcquireErrorKind, - HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceClients, - HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, - HlsSegmentCache, - HlsSessionHandle, MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, + HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, + HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, + HlsResourceFetchSource, HlsSegmentCache, HlsSessionHandle, MapCacheKey, MapCacheStatus, OriginMapFetchRef, + ProxyMapId, SegmentFetchPolicy, }; -use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use crate::{processing::parser::hls::origin_manifest::ParsedByteRange, utils::content_coding::DecodedHttpResponse}; use arc_swap::ArcSwap; use axum::http::HeaderMap; -use futures::{FutureExt, TryStreamExt}; +use futures::FutureExt; use log::debug; use reqwest::Client; -use std::{fmt, io, sync::Arc}; +use std::{fmt, sync::Arc}; use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore}; -use tokio_util::io::StreamReader; /// Shared context required to schedule EXT-X-MAP origin fetches without holding session locks. #[derive(Clone)] @@ -174,12 +173,7 @@ impl HlsMapWorkerPool { proxy_map_id_log ); - Some(MapFetchSnapshot { - proxy_map_id, - proxy_map_id_log, - cache_key, - fetch_ref, - }) + Some(MapFetchSnapshot { proxy_map_id, proxy_map_id_log, cache_key, fetch_ref }) } async fn fetch_one_map( @@ -216,8 +210,10 @@ impl HlsMapWorkerPool { response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::MapPermanentFailure { status: err.permanent_status(), }); - entry.status = - MapCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: err.permanent_status() }; + entry.status = MapCacheStatus::FailedPermanent { + failed_at_ms: finished_at_ms, + status: err.permanent_status(), + }; } } } @@ -248,10 +244,7 @@ impl HlsMapWorkerPool { reason: HlsAccessLeaseChannelUnavailableReason, ) -> usize { let proxy_session_id = session.read().await.proxy_session_id.clone(); - self.access_leases - .write() - .await - .mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) + self.access_leases.write().await.mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) } fn schedule_wake(self: &Arc, context: MapFetchContext, now_ms: u64) { @@ -396,7 +389,6 @@ async fn fetch_map_with_retries_into_cache( let session_log_id = context.session.read().await.proxy_session_id.0.clone(); let context = context.clone(); let snapshot = snapshot.clone(); - let commit_policy = policy.clone(); let policy_for_prepare = policy.clone(); let prepare_context = context.clone(); let cleanup_context = context.clone(); @@ -417,12 +409,11 @@ async fn fetch_map_with_retries_into_cache( } .boxed() }, - move |response, _attempt, guard| { + move |response, _attempt, body_deadline, guard| { let context = context.clone(); let snapshot = snapshot.clone(); - let policy = commit_policy.clone(); async move { - let commit_result = commit_map_response_into_cache(&context, &snapshot, &policy, response).await; + let commit_result = commit_map_response_into_cache(&context, &snapshot, response, body_deadline).await; let origin_work = finish_map_origin_attempt(context, guard).await; commit_result.map(|metadata| MapFetchCommit { content_length: metadata.size, @@ -438,24 +429,14 @@ async fn fetch_map_with_retries_into_cache( async fn commit_map_response_into_cache( context: &MapFetchContext, snapshot: &MapFetchSnapshot, - policy: &SegmentFetchPolicy, - response: reqwest::Response, + response: DecodedHttpResponse, + body_deadline: HlsOriginResourceBodyDeadline, ) -> Result { - let deadline = hls_object_body_deadline(policy.origin_segment_timeout_ms); - let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); context .segment_cache - .write_temp_and_commit_with_timeout(&snapshot.cache_key, stream_reader, deadline) + .write_temp_and_commit_with_deadline(&snapshot.cache_key, response.body, body_deadline.deadline()) .await - .map_err( - |err| { - if err.kind() == io::ErrorKind::TimedOut { - MapFetchError::Timeout - } else { - MapFetchError::cache_commit(&err) - } - }, - ) + .map_err(|err| MapFetchError::cache_body(&err)) } fn build_map_origin_headers( @@ -481,7 +462,8 @@ mod tests { }, }; use axum::http::{header, HeaderMap}; - use std::{sync::Arc, time::Duration}; + use flate2::{write::GzEncoder, Compression}; + use std::{io::Write, sync::Arc, time::Duration}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, @@ -526,7 +508,7 @@ mod tests { &HeaderMap::new(), Some(ParsedByteRange { offset: 10, length: 5 }), ) - .expect("headers should build"); + .expect("headers should build"); assert_eq!(headers.get(header::RANGE).expect("range"), "bytes=10-14"); assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); @@ -609,6 +591,97 @@ mod tests { server.await.expect("server joins"); } + #[tokio::test] + async fn gzip_fmp4_map_fetch_commits_identity_representation() { + const FMP4_MAP: &[u8] = b"\0\0\0\x18ftypisom\0\0\x02\0isomiso2"; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(FMP4_MAP).expect("gzip fixture should encode"); + let encoded_map = encoder.finish().expect("gzip fixture should finish"); + let encoded_len = encoded_map.len(); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buf = vec![0_u8; 2048]; + let read = socket.read(&mut buf).await.expect("request reads"); + let request = String::from_utf8_lossy(&buf[..read]); + assert!( + request.to_ascii_lowercase().contains("accept-encoding: identity"), + "map origin request must enforce identity" + ); + let headers = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp4\r\nContent-Encoding: gzip\r\nContent-Length: {encoded_len}\r\nConnection: close\r\n\r\n" + ); + socket.write_all(headers.as_bytes()).await.expect("response headers write"); + socket.write_all(&encoded_map).await.expect("response body writes"); + }); + let base_url = format!("http://{addr}/live/index.m3u8"); + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", &base_url); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path().to_path_buf())); + let worker = Arc::new(HlsMapWorkerPool::new(SegmentFetchPolicy { + retry_jitter_max_ms: 0, + origin_segment_timeout_ms: 1_000, + ..SegmentFetchPolicy::default() + })); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_map_access_lease(&worker, &proxy_session_id).await; + worker + .wake_scheduler( + MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::clone(&cache), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }, + 2, + ) + .await; + + for _ in 0..50 { + if matches!( + session.read().await.maps.values().next().map(|map| &map.status), + Some(MapCacheStatus::Ready { .. }) + ) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + let cache_key = { + let session = session.read().await; + let map = session.maps.values().next().expect("map"); + assert!(matches!( + &map.status, + MapCacheStatus::Ready { content_length, .. } + if *content_length == u64::try_from(FMP4_MAP.len()).expect("fixture length fits u64") + )); + map.cache_key.clone() + }; + let metadata = cache.metadata(&cache_key).await.expect("metadata reads").expect("map is committed"); + let cached_map = tokio::fs::read(metadata.path).await.expect("cached map reads"); + assert_eq!(cached_map, FMP4_MAP); + assert!(!cached_map.starts_with(&[0x1f, 0x8b])); + assert!(!cache.has_active_temp_files().await); + server.await.expect("server joins"); + } + #[tokio::test] async fn map_fetch_snapshot_uses_concrete_final_map_fetch_url() { let manifest = normal_manifest( @@ -638,10 +711,8 @@ mod tests { origin_io: None, }; - let snapshot = worker - .next_fetch_snapshot(&context, 2, &SegmentFetchPolicy::default()) - .await - .expect("map snapshot"); + let snapshot = + worker.next_fetch_snapshot(&context, 2, &SegmentFetchPolicy::default()).await.expect("map snapshot"); assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/init.mp4"); assert_eq!(snapshot.fetch_ref.byte_range, Some(ParsedByteRange { length: 5, offset: 10 })); diff --git a/backend/src/api/model/hls_cache/mod.rs b/backend/src/api/model/hls_cache/mod.rs index a1e5be3ee..15c6982f8 100644 --- a/backend/src/api/model/hls_cache/mod.rs +++ b/backend/src/api/model/hls_cache/mod.rs @@ -43,9 +43,9 @@ mod headers; mod ids; mod lease; mod lifecycle; +mod manager; mod manifest_commit; mod manifest_fetch; -mod manager; mod map; mod map_fetcher; mod observability; @@ -55,8 +55,8 @@ mod playback; mod prefetch; mod qos; mod refresh; -mod resource_fetch; mod renderer; +mod resource_fetch; mod response; mod segment_fetcher; mod segment_repair; @@ -87,22 +87,18 @@ pub use self::{ ids::{build_proxy_session_id, HlsSessionKey, ProxySessionId}, lease::{ new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, - HlsAccessLeaseId, HlsAccessLeaseIdleRelease, HlsAccessLeaseLifecycleSnapshot, - HlsAccessLeasePendingDeadline, HlsAccessLeaseResponseFlag, HlsAccessLeaseSessionSnapshot, - HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, - HlsFreshManifestRequiredReason, HlsPlaybackFamilyKey, + HlsAccessLeaseId, HlsAccessLeaseIdleRelease, HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, + HlsAccessLeaseResponseFlag, HlsAccessLeaseSessionSnapshot, HlsAccessLeaseState, HlsAccessLeaseStore, + HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsFreshManifestRequiredReason, HlsPlaybackFamilyKey, }, lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey, HlsLifecycleManager}, + manager::{exec_hls_lifecycle, HlsProxyManager}, manifest_commit::{ hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request, hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions, HlsCommittedManifestBody, }, - manifest_fetch::{ - classify_origin_manifest_status, LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, - RetryPolicy, - }, - manager::{exec_hls_lifecycle, HlsProxyManager}, + manifest_fetch::{classify_origin_manifest_status, LiveHlsOriginEntry, OriginManifestStatusClass, RetryPolicy}, map::{MapCacheStatus, MapEntry, OriginMapFetchRef, OriginMapKey, ProxyMapId}, map_fetcher::{HlsMapWorkerPool, MapFetchContext}, observability::{ @@ -130,21 +126,17 @@ pub use self::{ cold_start_retry_after_seconds, maybe_trigger_origin_refresh, trigger_origin_refresh_sync, HlsManifestCommitRequirement, OriginRefreshRequest, OriginRefreshState, }, - resource_fetch::{ - build_hls_origin_resource_headers, build_hls_origin_resource_headers_with_client_range, - classify_hls_resource_status, fetch_hls_origin_resource_response, log_hls_resource_attempt_started, - log_hls_resource_attempt_succeeded, log_hls_resource_fetch_failed, log_hls_resource_retry_scheduled, - log_hls_resource_timeout, retry_after_secs_from_ms, run_hls_origin_resource_retry_loop, - run_hls_origin_resource_retry_loop_with_attempt_prepare, HlsOriginByteRangeExpectation, - HlsOriginResourceAttemptCleanupFuture, HlsOriginResourceAttemptPrepareFuture, HlsOriginResourceClients, - HlsOriginResourceCommitFuture, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, - HlsResourceFetchAttempt, HlsResourceFetchKind, HlsResourceFetchLogContext, HlsResourceFetchLogStatus, - HlsResourceFetchSource, HlsResourceStatusClass, - }, renderer::{ renderer_candidate_window_proxy_seqs, HlsManifestRenderer, RenderError, RenderPolicy, RenderedManifest, RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, }, + resource_fetch::{ + build_hls_origin_resource_headers, build_hls_origin_resource_headers_with_client_range, + retry_after_secs_from_ms, run_hls_origin_resource_retry_loop_with_attempt_prepare, + HlsOriginByteRangeExpectation, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchAttempt, HlsResourceFetchKind, + HlsResourceFetchSource, + }, response::{ serve_hls_map_cache_outcome, serve_hls_map_cache_response, serve_hls_segment_cache_outcome, serve_hls_segment_cache_response, serve_hls_transient_object_cache_outcome, @@ -163,14 +155,12 @@ pub use self::{ TransientPassthroughReason, }, session_store::{ - HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsSessionHandle, HlsSessionStore, - HlsSessionStoreOutcome, + HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsSessionHandle, HlsSessionStore, HlsSessionStoreOutcome, }, timeline::{ - default_content_type_for_segment_ext, CacheAccessState, OriginSegmentFetchRef, OriginSegmentKey, - is_hls_provisioning_gap_segment, is_hls_provisioning_segment, SegmentCacheStatus, SegmentEntry, - TimelineMapError, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, - HLS_PROVISIONING_SEGMENT_DURATION_MS, + default_content_type_for_segment_ext, is_hls_provisioning_gap_segment, is_hls_provisioning_segment, + CacheAccessState, OriginSegmentFetchRef, OriginSegmentKey, SegmentCacheStatus, SegmentEntry, TimelineMapError, + HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, }, transient::{ @@ -183,9 +173,15 @@ pub use self::{ fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, hls_transient_origin_response, hls_transient_resource_fetch_kind, - is_hls_transient_full_object_cacheable_request, resolve_hls_transient_object_cache_action, - HlsTransientCacheCommitContext, HlsTransientObjectCacheAction, HlsTransientObjectCacheResolution, - HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, - HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, + is_hls_transient_full_object_cacheable_request, record_successful_transient_segment_fetch, + record_temporary_transient_segment_fetch_failure, resolve_hls_transient_object_cache_action, + HlsTransientCacheCommitContext, HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, + HlsTransientObjectCacheAction, HlsTransientObjectCacheResolution, HlsTransientObjectFetchFailure, + HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, + HlsTransientOriginIoGuard, }, }; +pub(crate) use self::{ + manifest_fetch::MAX_HLS_MANIFEST_BYTES, + observability::{log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource}, +}; diff --git a/backend/src/api/model/hls_cache/observability.rs b/backend/src/api/model/hls_cache/observability.rs index 067d1a4ee..3a84ccc4d 100644 --- a/backend/src/api/model/hls_cache/observability.rs +++ b/backend/src/api/model/hls_cache/observability.rs @@ -1,8 +1,80 @@ use super::{HlsAccessLeaseId, HlsSessionKey, ProxySessionId}; +use crate::utils::content_coding::ContentCodingObservation; +use log::debug; use sha2::{Digest, Sha256}; -use shared::utils::sanitize_sensitive_info; use std::sync::atomic::{AtomicU64, Ordering}; +/// Fixed HLS object classes allowed in content-coding diagnostics. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum HlsOriginContentCodingObjectKind { + Manifest, + Segment, + Map, + Key, + Part, + Other, +} + +impl HlsOriginContentCodingObjectKind { + const fn as_log_value(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Segment => "segment", + Self::Map => "map", + Self::Key => "key", + Self::Part => "part", + Self::Other => "other", + } + } +} + +/// Fixed Tuliprox HLS stacks allowed in content-coding diagnostics. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum HlsOriginContentCodingSource { + Legacy, + Shared, +} + +impl HlsOriginContentCodingSource { + const fn as_log_value(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Shared => "shared", + } + } +} + +/// Logs a prepared origin-coding normalization using fixed or numeric fields only. +pub(crate) fn log_hls_origin_content_coding( + observation: ContentCodingObservation, + object_kind: HlsOriginContentCodingObjectKind, + range_requested: bool, + source: HlsOriginContentCodingSource, +) { + debug!( + "HLS origin content coding normalization prepared: {}", + hls_origin_content_coding_log_fields(observation, object_kind, range_requested, source) + ); +} + +fn hls_origin_content_coding_log_fields( + observation: ContentCodingObservation, + object_kind: HlsOriginContentCodingObjectKind, + range_requested: bool, + source: HlsOriginContentCodingSource, +) -> String { + let content_length = observation.content_length.map_or_else(|| "unknown".to_string(), |value| value.to_string()); + format!( + "object_kind={} content_encoding={} status={} requested_accept_encoding=identity decoded_to_identity=true content_length={} range_requested={} source={}", + object_kind.as_log_value(), + observation.content_encoding, + observation.status.as_u16(), + content_length, + range_requested, + source.as_log_value() + ) +} + /// In-memory counters for the live HLS cache path. #[derive(Debug, Default)] pub struct HlsCacheMetrics { @@ -120,7 +192,7 @@ pub fn safe_user_session_token(session_token: &str) -> String { short_hash(sessi pub fn safe_proxy_session_id(proxy_session_id: &ProxySessionId) -> String { short_hash(&proxy_session_id.0) } -pub fn safe_origin_log_value(value: impl AsRef) -> String { sanitize_sensitive_info(value.as_ref()).into_owned() } +pub fn safe_origin_log_value(value: impl AsRef) -> String { format!("origin#{}", short_hash(value.as_ref())) } fn increment(counter: &AtomicU64, count: u64) { counter.fetch_add(count, Ordering::Relaxed); } @@ -134,12 +206,20 @@ fn short_hash(value: &str) -> String { #[cfg(test)] mod tests { - use super::{safe_origin_log_value, safe_proxy_session_id, safe_session_key, HlsCacheMetrics}; - use crate::api::model::{HlsSessionKey, ProxySessionId}; + use super::{ + hls_origin_content_coding_log_fields, safe_origin_log_value, safe_proxy_session_id, safe_session_key, + HlsCacheMetrics, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, + }; + use crate::{ + api::model::{HlsSessionKey, ProxySessionId}, + utils::content_coding::ContentCodingObservation, + }; + use reqwest::StatusCode; #[test] fn safe_log_helpers_do_not_emit_credentials_or_full_proxy_session_id() { let sanitized = safe_origin_log_value("http://user:password@example.com/live/user/password/123.ts"); + assert!(sanitized.starts_with("origin#")); assert!(!sanitized.contains("user:password")); assert!(!sanitized.contains("/user/password/")); @@ -171,4 +251,47 @@ mod tests { assert_eq!(snapshot.prefetch_queued, 2); assert_eq!(snapshot.prefetch_skipped, 3); } + + #[test] + fn content_coding_log_fields_are_fixed_redacted_and_complete() { + let fields = hls_origin_content_coding_log_fields( + ContentCodingObservation { + content_encoding: "multiple", + status: StatusCode::OK, + content_length: Some(321), + }, + HlsOriginContentCodingObjectKind::Manifest, + false, + HlsOriginContentCodingSource::Legacy, + ); + + assert_eq!( + fields, + "object_kind=manifest content_encoding=multiple status=200 requested_accept_encoding=identity decoded_to_identity=true content_length=321 range_requested=false source=legacy" + ); + for sensitive_marker in ["http", "?", "cookie", "manifest-body", "signed-token"] { + assert!(!fields.contains(sensitive_marker)); + } + + let unknown_length = hls_origin_content_coding_log_fields( + ContentCodingObservation { + content_encoding: "gzip", + status: StatusCode::PARTIAL_CONTENT, + content_length: None, + }, + HlsOriginContentCodingObjectKind::Segment, + true, + HlsOriginContentCodingSource::Shared, + ); + assert!(unknown_length.contains("content_length=unknown")); + assert!(unknown_length.contains("range_requested=true source=shared")); + + let part = hls_origin_content_coding_log_fields( + ContentCodingObservation { content_encoding: "br", status: StatusCode::OK, content_length: Some(17) }, + HlsOriginContentCodingObjectKind::Part, + false, + HlsOriginContentCodingSource::Shared, + ); + assert!(part.starts_with("object_kind=part content_encoding=br status=200 ")); + } } diff --git a/backend/src/api/model/hls_cache/origin.rs b/backend/src/api/model/hls_cache/origin.rs index 0fadab535..d49c94994 100644 --- a/backend/src/api/model/hls_cache/origin.rs +++ b/backend/src/api/model/hls_cache/origin.rs @@ -5,8 +5,12 @@ use crate::{ }; use log::debug; use shared::utils::sanitize_sensitive_info; -use std::{fmt, net::SocketAddr, sync::Arc}; -use std::time::{Duration, Instant}; +use std::{ + fmt, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; use tokio::{ sync::{Mutex, Notify}, time::timeout, @@ -16,15 +20,26 @@ const HLS_ACCOUNT_OVERLAP_FALLBACK_TARGET_DURATION_MS: u64 = 15_000; const HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK: Duration = Duration::from_millis(25); /// Stable source metadata for one shared live-HLS session. +/// +/// Session identity is derived only from `input_id` and the immutable input +/// origin ID in `stream_ref`; target IDs, virtual IDs, and resolved origin URLs +/// are deliberately excluded. #[derive(Debug, Clone, Eq, PartialEq)] pub struct HlsOriginSource { + /// Internal ID of the configured Tuliprox input. pub input_id: u16, + /// Input name used to resolve the current origin account or alias. pub input_name: Arc, + /// Exact, non-empty origin/provider stream ID captured before target mapping. pub stream_ref: String, pub source_kind: HlsOriginSourceKind, } impl HlsOriginSource { + /// Creates source metadata from an immutable input-stream identity. + /// + /// `stream_ref` must be the origin/provider ID, never a target-specific or + /// virtual ID. pub fn new( input_id: u16, input_name: Arc, @@ -129,11 +144,7 @@ impl HlsAccountOverlapTiming { } pub fn reservation_ttl_secs(self) -> u64 { - self.hard_active_window_ms - .saturating_add(self.soft_active_window_ms) - .saturating_add(999) - / 1_000 - + 1 + self.hard_active_window_ms.saturating_add(self.soft_active_window_ms).saturating_add(999) / 1_000 + 1 } } @@ -616,9 +627,7 @@ async fn reserve_hls_origin_account_io_slot( if now >= deadline { return Err(HlsBoundAccountAcquireErrorKind::WaitTimedOut); } - let wait_for = deadline - .saturating_duration_since(now) - .min(HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK); + let wait_for = deadline.saturating_duration_since(now).min(HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK); tokio::select! { () = notify.notified() => {} () = tokio::time::sleep(wait_for) => {} @@ -846,6 +855,15 @@ mod tests { assert_eq!(direct.session_key().stable_value(), "input:7|hls|80510"); } + #[test] + fn origin_source_preserves_alphanumeric_input_stream_id() { + let source = + HlsOriginSource::new(7, Arc::from("input"), "m3u-channel_A42", HlsOriginSourceKind::M3uMediaPlaylist); + + assert_eq!(source.stream_ref, "m3u-channel_A42"); + assert_eq!(source.session_key().stable_value(), "input:7|hls|m3u-channel_A42"); + } + #[test] fn account_overlap_timing_uses_target_duration_or_fallback() { let timing = HlsAccountOverlapTiming::from_target_duration_secs(Some(12)); diff --git a/backend/src/api/model/hls_cache/prefetch.rs b/backend/src/api/model/hls_cache/prefetch.rs index f36e8c6a7..8d85ff671 100644 --- a/backend/src/api/model/hls_cache/prefetch.rs +++ b/backend/src/api/model/hls_cache/prefetch.rs @@ -222,8 +222,7 @@ impl HlsSession { return report; } - let tail_index = - known_sequences.len().saturating_sub(1).saturating_sub(self.initial_prefetch_gap_segments); + let tail_index = known_sequences.len().saturating_sub(1).saturating_sub(self.initial_prefetch_gap_segments); let render_window_len = known_sequences.len().min(6).min(tail_index.saturating_add(1)); let render_start_index = tail_index.saturating_add(1).saturating_sub(render_window_len); @@ -260,13 +259,7 @@ impl HlsSession { pub fn configure_segment_prefetch_queue(&mut self, max_prefetch_depth: usize) { for proxy_seq in self.segment_prefetch_queue.set_max_prefetch_depth(max_prefetch_depth) { if let Some(entry) = self.segments.get_mut(&proxy_seq) { - if matches!( - entry.status, - SegmentCacheStatus::Queued { - priority: SegmentFetchPriority::Prefetch, - .. - } - ) { + if matches!(entry.status, SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, .. }) { entry.status = SegmentCacheStatus::Discovered; } } @@ -395,13 +388,7 @@ mod tests { session.configure_segment_prefetch_queue(1); - assert!(matches!( - session.segments.get(&4).expect("trimmed segment").status, - SegmentCacheStatus::Discovered - )); - assert!(matches!( - session.segments.get(&5).expect("trimmed segment").status, - SegmentCacheStatus::Discovered - )); + assert!(matches!(session.segments.get(&4).expect("trimmed segment").status, SegmentCacheStatus::Discovered)); + assert!(matches!(session.segments.get(&5).expect("trimmed segment").status, SegmentCacheStatus::Discovered)); } } diff --git a/backend/src/api/model/hls_cache/qos.rs b/backend/src/api/model/hls_cache/qos.rs index 625782571..df10d566f 100644 --- a/backend/src/api/model/hls_cache/qos.rs +++ b/backend/src/api/model/hls_cache/qos.rs @@ -1,8 +1,5 @@ use super::{HlsAccessLeaseId, ProxySessionId}; -use crate::{ - api::model::StreamMeterHandle, - model::AppConfig, -}; +use crate::{api::model::StreamMeterHandle, model::AppConfig}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::RwLock; @@ -18,9 +15,7 @@ impl HlsQosRuntimeConfig { let Some(reverse_proxy) = config.reverse_proxy.as_ref() else { return Self::default(); }; - Self { - live_metering_enabled: reverse_proxy.stream.as_ref().is_some_and(|stream| stream.metrics_enabled), - } + Self { live_metering_enabled: reverse_proxy.stream.as_ref().is_some_and(|stream| stream.metrics_enabled) } } } @@ -44,13 +39,12 @@ struct HlsAccessLeaseQosState { } impl HlsAccessLeaseQosState { - fn registration(&self, emit_connect_record: bool, register_meter: Option>) -> HlsQosRegistration { - HlsQosRegistration { - meter_uid: self.meter_uid, - meter: self.meter.clone(), - register_meter, - emit_connect_record, - } + fn registration( + &self, + emit_connect_record: bool, + register_meter: Option>, + ) -> HlsQosRegistration { + HlsQosRegistration { meter_uid: self.meter_uid, meter: self.meter.clone(), register_meter, emit_connect_record } } } @@ -74,11 +68,7 @@ impl HlsQosRegistry { let (meter_uid, meter) = meter_init.map_or((0, None), |init| (init.meter_uid, Some(init.meter))); let register_meter = meter.clone(); - let state = HlsAccessLeaseQosState { - proxy_session_id: proxy_session_id.clone(), - meter_uid, - meter, - }; + let state = HlsAccessLeaseQosState { proxy_session_id: proxy_session_id.clone(), meter_uid, meter }; let registration = state.registration(true, register_meter); states.insert(lease_id.clone(), state); registration @@ -94,10 +84,7 @@ impl HlsQosRegistry { pub async fn remove_access_leases(&self, lease_ids: &[HlsAccessLeaseId]) -> usize { let mut states = self.states.write().await; - lease_ids - .iter() - .filter(|lease_id| states.remove(*lease_id).is_some()) - .count() + lease_ids.iter().filter(|lease_id| states.remove(*lease_id).is_some()).count() } pub async fn remove_proxy_session_state(&self, proxy_session_id: &ProxySessionId) -> usize { @@ -122,8 +109,7 @@ impl HlsQosRegistry { #[cfg(test)] mod tests { use super::{HlsQosMeterInit, HlsQosRegistry}; - use crate::api::model::{EventManager, StreamMeterHandle}; - use crate::api::model::{HlsAccessLeaseId, ProxySessionId}; + use crate::api::model::{EventManager, HlsAccessLeaseId, ProxySessionId, StreamMeterHandle}; use std::sync::Arc; #[tokio::test] diff --git a/backend/src/api/model/hls_cache/refresh.rs b/backend/src/api/model/hls_cache/refresh.rs index 80333bce1..3a59459ee 100644 --- a/backend/src/api/model/hls_cache/refresh.rs +++ b/backend/src/api/model/hls_cache/refresh.rs @@ -1,6 +1,6 @@ use super::{ - begin_hls_origin_account_io, finish_hls_origin_account_io, is_hls_provisioning_gap_segment, - is_hls_provisioning_segment, + begin_hls_origin_account_io, finish_hls_origin_account_io, hls_origin_headers_with_provider_session, + is_hls_provisioning_gap_segment, is_hls_provisioning_segment, manifest_fetch::{ commit_error_to_fetch_error, evaluate_manifest_origin_quality_with_mode, fetch_hls_origin_manifest_request, fetched_effective_manifest_host, log_hls_manifest_initial_selected, manifest_host_switch_failure_threshold, @@ -12,12 +12,11 @@ use super::{ OriginManifestFetchError, RetryPolicy, }, safe_hls_access_lease_id, safe_origin_log_value, safe_proxy_session_id, safe_session_key, - hls_origin_headers_with_provider_session, sanitized_hls_origin_headers, - HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsFreshManifestRequiredReason, HlsManifestRenderer, - HlsManifestTemporaryFailureKind, HlsManifestTemporaryFailureTransition, HlsMapWorkerPool, HlsOriginIoContext, - HlsOriginWorkClass, HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, - HlsSessionMode, MapFetchContext, RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, - SegmentFetchContext, TransientPassthroughReason, + sanitized_hls_origin_headers, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, + HlsFreshManifestRequiredReason, HlsManifestRenderer, HlsManifestTemporaryFailureKind, + HlsManifestTemporaryFailureTransition, HlsMapWorkerPool, HlsOriginIoContext, HlsOriginWorkClass, HlsProxyManager, + HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, HlsSessionMode, MapFetchContext, + RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, SegmentFetchContext, TransientPassthroughReason, }; use crate::{ model::{AppConfig, HlsManifestRecoveryBurstConfig, ReverseProxyDisabledHeaderConfig, StripConfig}, @@ -38,8 +37,7 @@ use crate::{ use axum::http::HeaderMap; use log::{debug, info, warn}; use reqwest::Client; -use shared::model::HlsStripMode; -use shared::utils::sanitize_sensitive_info; +use shared::{model::HlsStripMode, utils::sanitize_sensitive_info}; use std::sync::Arc; use url::Url; @@ -404,7 +402,7 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ "HLS origin manifest refresh completed: session={} proxy_session_id={} result=failed error={}", safe_session_key(&session.key), safe_proxy_session_id(&session.proxy_session_id), - safe_origin_log_value(format!("{err:?}")) + err.log_label() ); ( false, @@ -513,11 +511,7 @@ async fn mark_fresh_manifest_commit_failed_access_leases( let marked = origin_io .app_state .hls_proxy - .mark_access_leases_channel_unavailable_for_session( - &proxy_session_id, - failed_at_ms, - unavailable_reason, - ) + .mark_access_leases_channel_unavailable_for_session(&proxy_session_id, failed_at_ms, unavailable_reason) .await; if marked > 0 { debug!( @@ -574,19 +568,33 @@ fn manifest_temporary_failure_kind(err: &OriginManifestFetchError) -> Option None, + | OriginManifestFetchError::ProviderUnavailable(_) + | OriginManifestFetchError::ContentCoding(_) + | OriginManifestFetchError::ContentDecoding { .. } + | OriginManifestFetchError::DecodedBodyLimitExceeded { .. } + | OriginManifestFetchError::InvalidUtf8 { .. } => None, } } fn manifest_hard_fetch_error(err: &OriginManifestFetchError) -> bool { match err { - OriginManifestFetchError::PermanentStatus(_) | OriginManifestFetchError::NonRetryableStatus(_) => true, + OriginManifestFetchError::PermanentStatus(_) + | OriginManifestFetchError::NonRetryableStatus(_) + | OriginManifestFetchError::ContentCoding( + crate::utils::content_coding::ContentCodingError::InvalidHeader + | crate::utils::content_coding::ContentCodingError::Unsupported(_) + | crate::utils::content_coding::ContentCodingError::EncodedPartialContent, + ) + | OriginManifestFetchError::DecodedBodyLimitExceeded { .. } + | OriginManifestFetchError::InvalidUtf8 { .. } => true, OriginManifestFetchError::ProviderUnavailable(kind) => !kind.is_retryable_resource_failure(), OriginManifestFetchError::RetryableStatus(_, _) | OriginManifestFetchError::RetryExhausted | OriginManifestFetchError::Request(_) | OriginManifestFetchError::Redirect(_) - | OriginManifestFetchError::Timeout => false, + | OriginManifestFetchError::Timeout + | OriginManifestFetchError::ContentCoding(crate::utils::content_coding::ContentCodingError::PrefixRead(_)) + | OriginManifestFetchError::ContentDecoding { .. } => false, } } @@ -661,7 +669,7 @@ async fn finish_refresh_failure(request: &OriginRefreshRequest, err: OriginManif "HLS origin manifest refresh completed: session={} proxy_session_id={} result=failed error={}", safe_session_key(&session.key), safe_proxy_session_id(&session.proxy_session_id), - safe_origin_log_value(format!("{err:?}")) + err.log_label() ); (request.manifest_commit_requirement.fresh_reason(), temporary_manifest_failure_reason) }; @@ -731,9 +739,7 @@ async fn fetch_and_commit_manifest_with_policy( ) -> Result { let fetch_context = manifest_fetch_context(request); let fetched = - fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&fetch_context)) - .await? - .with_attempts(1); + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&fetch_context)).await?; let acceptance_mode = request.manifest_commit_requirement.acceptance_mode(); let selected_report = score_hls_manifest_candidate_for_selection_log(&fetch_context, &fetched, acceptance_mode).await; @@ -1057,10 +1063,7 @@ fn evaluate_manifest_acceptance( } } -fn mark_manifest_handoff_discontinuity_if_needed( - session: &mut super::HlsSession, - quality: &HlsManifestOriginQuality, -) { +fn mark_manifest_handoff_discontinuity_if_needed(session: &mut super::HlsSession, quality: &HlsManifestOriginQuality) { if !quality.requires_handoff_discontinuity { return; } @@ -1264,7 +1267,8 @@ fn commit_normal_manifest( let provisioning_handoff = session.pending_handoff_discontinuity_sequence.is_some() && session.segments.values().any(is_hls_provisioning_segment); let segment_durations = manifest.segments.iter().map(|segment| segment.duration_ms).collect::>(); - session.initial_prefetch_gap_segments = initial_hls_strip_segments_for_durations(&request.strip, &segment_durations); + session.initial_prefetch_gap_segments = + initial_hls_strip_segments_for_durations(&request.strip, &segment_durations); session .apply_origin_manifest(manifest) .map_err(|err| HlsManifestCommitError::TimelineRejected { reason: HlsManifestRejectLogReason::from(err) })?; @@ -1511,41 +1515,49 @@ fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into mod tests { use super::{ super::manifest_fetch::{ - classify_origin_manifest_status, hls_manifest_redirect_host, + classify_origin_manifest_status, fetch_hls_origin_manifest_request, hls_manifest_redirect_host, manifest_host_switch_failure_threshold_for_strip_segments, origin_highwater_policy_limit, refresh_from_live_hls_entrypoint_with_retries, resolved_hls_manifest_request_url_from_input, retry_after_delay_ms, retry_hls_origin_manifest_recovery_chain, score_hls_manifest_recovery_candidate as score_manifest_recovery_candidate, FetchedOriginManifest, HlsManifestCommitAcceptanceMode, HlsManifestCommitError, HlsManifestOriginQualityScore, - HlsManifestRejectLogReason, HlsManifestSequenceRelation, LiveHlsOriginEntry, - ManifestRecoverySelectionLogPhase, OriginManifestFetchError, OriginManifestStatusClass, RetryPolicy, + HlsManifestRejectLogReason, HlsManifestSequenceRelation, HlsOriginManifestFetchContext, + HlsOriginManifestFetchRequest, LiveHlsOriginEntry, ManifestRecoverySelectionLogPhase, + OriginManifestFetchError, OriginManifestStatusClass, RetryPolicy, }, build_manifest_refresh_timing, commit_fetched_manifest, compute_origin_refresh_interval_ms, fetched_effective_manifest_host, format_millis_as_seconds, format_optional_millis_as_seconds, - manifest_fetch_context, manifest_hard_fetch_error, manifest_progress_from_highwater, mark_origin_refresh_started, - manifest_temporary_failure_kind, record_pinned_host_recovery_chain_failed, request_error_indicates_timeout, - transient_reason_log_fields, HlsManifestAcceptanceDecision, HlsManifestCommitRequirement, HlsManifestProgress, - trigger_origin_refresh_sync, OriginRefreshRequest, OriginRefreshState, + manifest_fetch_context, manifest_hard_fetch_error, manifest_progress_from_highwater, + manifest_temporary_failure_kind, mark_origin_refresh_started, record_pinned_host_recovery_chain_failed, + request_error_indicates_timeout, transient_reason_log_fields, trigger_origin_refresh_sync, + HlsManifestAcceptanceDecision, HlsManifestCommitRequirement, HlsManifestProgress, OriginRefreshRequest, + OriginRefreshState, }; use crate::{ api::model::{ maybe_trigger_origin_refresh, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, - HlsFreshManifestRequiredReason, HlsMapWorkerPool, HlsOriginAccountBinding, HlsPlaybackFamilyKey, - HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSession, - HlsSessionKey, HlsSessionMode, HlsBoundAccountAcquireErrorKind, TimelineMapError, - TransientPassthroughReason, + HlsBoundAccountAcquireErrorKind, HlsFreshManifestRequiredReason, HlsMapWorkerPool, HlsOriginAccountBinding, + HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, + HlsSession, HlsSessionKey, HlsSessionMode, TimelineMapError, TransientPassthroughReason, }, model::{ - AppConfig, Config, ConfigProvider, HlsManifestRecoveryBurstConfig, - HlsSegmentRepairConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, + AppConfig, Config, ConfigProvider, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, + ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, }, + utils::content_coding::{ContentCoding, ContentCodingError}, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}; - use shared::model::{ConfigPaths, ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, ProviderUrlSelectionPolicy}; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, + use shared::model::{ + ConfigPaths, ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, + ProviderUrlSelectionPolicy, + }; + use std::{ + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, @@ -1736,6 +1748,72 @@ mod tests { ))); } + #[test] + fn manifest_content_failures_have_explicit_hard_and_temporary_classification() { + let errors = vec![ + ( + "invalid content-coding header", + OriginManifestFetchError::ContentCoding(ContentCodingError::InvalidHeader), + true, + ), + ( + "unsupported content coding", + OriginManifestFetchError::ContentCoding(ContentCodingError::Unsupported("unknown".to_string())), + true, + ), + ( + "encoded partial content", + OriginManifestFetchError::ContentCoding(ContentCodingError::EncodedPartialContent), + true, + ), + ( + "content-coding prefix read", + OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(io::Error::other( + "prefix read failed", + ))), + false, + ), + ( + "gzip decoding", + OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Gzip }, + false, + ), + ( + "deflate decoding", + OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Deflate }, + false, + ), + ( + "brotli decoding", + OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Brotli }, + false, + ), + ( + "zstandard decoding", + OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Zstd }, + false, + ), + ( + "decoded body limit", + OriginManifestFetchError::DecodedBodyLimitExceeded { limit: 1024 }, + true, + ), + ( + "invalid utf-8", + OriginManifestFetchError::InvalidUtf8 { valid_up_to: 7, error_len: Some(1) }, + true, + ), + ]; + + for (label, error, expected_hard) in errors { + assert_eq!(manifest_hard_fetch_error(&error), expected_hard, "unexpected hard classification: {label}"); + assert!( + manifest_temporary_failure_kind(&error).is_none(), + "content failure must not affect the temporary host-switch counter: {label}" + ); + } + } + #[test] fn request_error_timeout_detection_matches_global_helper_wording() { assert!(request_error_indicates_timeout("Request timed out and no retries left")); @@ -2125,9 +2203,8 @@ mod tests { let session = test_session(); session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ColdStart, - }; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; assert!(mark_origin_refresh_started(&mut request, 1_000).await); assert!(session.read().await.origin_refresh.in_flight); @@ -2560,6 +2637,7 @@ mod tests { struct TestOriginServer { base_url: String, requests: Arc>>, + raw_requests: Arc>>, task: tokio::task::JoinHandle<()>, } @@ -2574,12 +2652,15 @@ mod tests { let addr = listener.local_addr().expect("local addr"); let requests = Arc::new(Mutex::new(Vec::new())); let requests_for_task = Arc::clone(&requests); + let raw_requests = Arc::new(Mutex::new(Vec::new())); + let raw_requests_for_task = Arc::clone(&raw_requests); let task = tokio::spawn(async move { loop { let Ok((mut socket, _)) = listener.accept().await else { break; }; let requests = Arc::clone(&requests_for_task); + let raw_requests = Arc::clone(&raw_requests_for_task); let handler = Arc::clone(&handler); tokio::spawn(async move { let mut buf = vec![0_u8; 4096]; @@ -2599,7 +2680,7 @@ mod tests { return; } } - let request = String::from_utf8_lossy(&buf[..used]); + let request = String::from_utf8_lossy(&buf[..used]).into_owned(); let path = request .lines() .next() @@ -2607,6 +2688,7 @@ mod tests { .unwrap_or("/") .to_string(); requests.lock().await.push(path.clone()); + raw_requests.lock().await.push(request); let (status, headers, body) = handler(path); let reason = match status { 200 => "OK", @@ -2632,7 +2714,14 @@ mod tests { }); } }); - TestOriginServer { base_url: format!("http://{addr}"), requests, task } + TestOriginServer { base_url: format!("http://{addr}"), requests, raw_requests, task } + } + + fn request_header_value<'a>(request: &'a str, expected_name: &str) -> Option<&'a str> { + request.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) + }) } fn no_delay_policy() -> RetryPolicy { RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 } } @@ -2720,6 +2809,281 @@ mod tests { fn manifest_body() -> String { "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string() } + #[tokio::test] + async fn shared_initial_manifest_decoder_failure_retries_until_success() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("shared initial manifest should retry decoder failures"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 3); + assert_eq!(server.requests.lock().await.len(), 3); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + } + + #[tokio::test] + async fn shared_initial_manifest_waits_for_next_attempt_base_delay() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: RetryPolicy { delays_ms: [0, 100, 250, 500, 750], jitter_max_ms: 0 }, + }; + let started_at = std::time::Instant::now(); + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("second logical attempt should succeed after its base delay"); + + assert_eq!(fetched.attempts, 2); + assert_eq!(server.requests.lock().await.len(), 2); + assert!(started_at.elapsed() >= std::time::Duration::from_millis(100)); + } + + #[tokio::test] + async fn shared_initial_manifest_automatic_cross_origin_redirect_keeps_identity_and_scrubs_credentials() { + let target = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let target_url = format!("{}/final/manifest.m3u8", target.base_url); + let expected_target_url = target_url.clone(); + let redirect = + spawn_test_origin(Arc::new(move |_path| (302, vec![("Location", target_url.clone())], String::new()))) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", redirect.base_url)).expect("entry url"); + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer origin-secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=origin-secret")); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers, + client: reqwest::Client::builder().no_proxy().build().expect("client"), + no_redirect_client: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("automatic redirect should return the decoded manifest"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 1); + assert_eq!(fetched.final_manifest_url, expected_target_url); + let redirect_requests = redirect.raw_requests.lock().await; + let target_requests = target.raw_requests.lock().await; + assert_eq!(redirect_requests.len(), 1); + assert_eq!(target_requests.len(), 1); + assert_eq!(request_header_value(&redirect_requests[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&target_requests[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&redirect_requests[0], "authorization"), Some("Bearer origin-secret")); + assert_eq!(request_header_value(&redirect_requests[0], "cookie"), Some("sid=origin-secret")); + assert!(request_header_value(&target_requests[0], "authorization").is_none()); + assert!(request_header_value(&target_requests[0], "cookie").is_none()); + } + + #[tokio::test] + async fn shared_initial_manifest_budget_covers_status_decoder_redirect_and_success() { + let target_hits = Arc::new(AtomicUsize::new(0)); + let target_hits_for_handler = Arc::clone(&target_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path == "/live/user/pass/12345.m3u8" { + return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); + } + if path == "/live/play/once/12345" { + return match target_hits_for_handler.fetch_add(1, Ordering::SeqCst) { + 0 => (500, Vec::new(), "temporary".to_string()), + 1 => (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()), + _ => (200, Vec::new(), manifest_body()), + }; + } + (404, Vec::new(), String::new()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: true, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("third logical attempt should succeed"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 3); + assert_eq!( + *server.requests.lock().await, + vec![ + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + ] + ); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + } + + #[tokio::test] + async fn shared_initial_manifest_decoder_failures_stop_at_attempt_budget() { + let server = spawn_test_origin(Arc::new(|_path| { + (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let retry_policy = no_delay_policy(); + let expected_attempts = retry_policy.attempt_count(); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy, + }; + + let error = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect_err("decoder failures must exhaust the logical attempt budget"); + + assert!(matches!(error, OriginManifestFetchError::ContentDecoding { .. })); + assert_eq!(server.requests.lock().await.len(), expected_attempts); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + } + + #[tokio::test] + async fn shared_refresh_metrics_use_successful_manifest_attempt_count() { + let manifest_hits = Arc::new(AtomicUsize::new(0)); + let manifest_hits_for_handler = Arc::clone(&manifest_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path != "/live/user/pass/12345.m3u8" { + return (404, Vec::new(), String::new()); + } + if manifest_hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let session = test_session(); + let mut request = test_origin_refresh_request(session); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + + assert!(trigger_origin_refresh_sync(request).await); + + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.refresh_started, 1); + assert_eq!(snapshot.refresh_completed, 1); + assert_eq!(snapshot.refresh_retried, 2); + assert_eq!(snapshot.refresh_failed, 0); + let manifest_requests = server + .raw_requests + .lock() + .await + .iter() + .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) + .cloned() + .collect::>(); + assert_eq!(manifest_requests.len(), 3); + assert!(manifest_requests + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + } + #[tokio::test] async fn manifest_retry_starts_at_entrypoint_after_redirect_failure() { let redirect_hits = Arc::new(AtomicUsize::new(0)); @@ -2879,14 +3243,26 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(10)).await; } - let first_requests = first.requests.lock().await; - let second_requests = second.requests.lock().await; let first_manifest_requests = - first_requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); let second_manifest_requests = - second_requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); assert_eq!(first_manifest_requests, 1); assert_eq!(second_manifest_requests, 1); + for server in [&first, &second] { + let manifest_requests = server + .raw_requests + .lock() + .await + .iter() + .filter(|request| { + request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 ")) + }) + .cloned() + .collect::>(); + assert_eq!(manifest_requests.len(), 1); + assert_eq!(request_header_value(&manifest_requests[0], "accept-encoding"), Some("identity")); + } let session = session.read().await; assert_eq!(session.key.stable_value(), initial_session_key); assert_eq!(session.proxy_session_id, initial_proxy_session_id); @@ -3043,9 +3419,8 @@ mod tests { score_manifest_recovery_candidate(&session, &same_host_next, &fetch_context).expect("score").quality.score, HlsManifestOriginQualityScore::SameHostNextSequence ); - let other_host_score = score_manifest_recovery_candidate(&session, &other_host_next, &fetch_context) - .expect("score") - .quality; + let other_host_score = + score_manifest_recovery_candidate(&session, &other_host_next, &fetch_context).expect("score").quality; assert_eq!(other_host_score.score, HlsManifestOriginQualityScore::OtherHostNextSequence); assert!(other_host_score.requires_handoff_discontinuity); } diff --git a/backend/src/api/model/hls_cache/renderer.rs b/backend/src/api/model/hls_cache/renderer.rs index dadc09a4a..97bc4ee97 100644 --- a/backend/src/api/model/hls_cache/renderer.rs +++ b/backend/src/api/model/hls_cache/renderer.rs @@ -64,10 +64,7 @@ pub enum RenderedManifestStoreOutcome { #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum RenderedManifestStoreRejectReason { - RegressiveMediaSequence { - previous_first_proxy_seq: u64, - candidate_first_proxy_seq: u64, - }, + RegressiveMediaSequence { previous_first_proxy_seq: u64, candidate_first_proxy_seq: u64 }, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -164,10 +161,7 @@ fn select_window(session: &HlsSession, render_gap_segments: usize) -> Option Option u32 { } fn window_contains_provisioning_segment(session: &HlsSession, window: &[u64]) -> bool { - window - .iter() - .filter_map(|proxy_seq| session.segments.get(proxy_seq)) - .any(is_local_provisioning_segment) + window.iter().filter_map(|proxy_seq| session.segments.get(proxy_seq)).any(is_local_provisioning_segment) } fn window_contains_origin_segment(session: &HlsSession, window: &[u64]) -> bool { @@ -380,7 +369,6 @@ fn format_duration_ms(duration_ms: u64) -> String { format!("{}.{:03}", duration #[cfg(test)] mod tests { - use shared::model::HlsStripMode; use super::{ HlsManifestRenderer, RenderError, RenderPolicy, RenderedManifest, RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, @@ -390,6 +378,7 @@ mod tests { model::StripConfig, processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, }; + use shared::model::HlsStripMode; const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; @@ -433,9 +422,8 @@ mod tests { fn rendering_an_empty_internal_window_does_not_panic() { let session = session(); - let rendered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - super::render_window(&session, &[], 0, 0) - })); + let rendered = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| super::render_window(&session, &[], 0, 0))); assert!(rendered.is_ok()); } diff --git a/backend/src/api/model/hls_cache/resource_fetch.rs b/backend/src/api/model/hls_cache/resource_fetch.rs index 6083a2e5b..a842dd250 100644 --- a/backend/src/api/model/hls_cache/resource_fetch.rs +++ b/backend/src/api/model/hls_cache/resource_fetch.rs @@ -1,9 +1,17 @@ use super::{ - append_hls_provider_session_headers, force_identity_without_range, hls_object_body_deadline, safe_origin_log_value, - safe_proxy_session_id, scrub_hls_origin_headers, HlsBoundAccountAcquireErrorKind, ProxySessionId, - SegmentFetchPolicy, + append_hls_provider_session_headers, cache::hls_cache_object_limit_from_io, force_identity_without_range, + hls_object_body_deadline, log_hls_origin_content_coding, safe_origin_log_value, safe_proxy_session_id, + scrub_hls_origin_headers, HlsBoundAccountAcquireErrorKind, HlsOriginContentCodingObjectKind, + HlsOriginContentCodingSource, ProxySessionId, SegmentFetchPolicy, +}; +use crate::{ + processing::parser::hls::origin_manifest::ParsedByteRange, + utils::content_coding::{ + apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, + is_http_body_transport_error, ContentCodingDetection, ContentCodingError, ContentDecodingIoError, + DecodedHttpResponse, OutboundContentCodingPolicy, + }, }; -use crate::processing::parser::hls::origin_manifest::ParsedByteRange; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use futures::{future::BoxFuture, FutureExt}; use log::{debug, warn}; @@ -19,14 +27,14 @@ const MAX_MANUAL_REDIRECTS: usize = 10; const STORAGE_FULL_RAW_OS_ERRORS: &[i32] = &[28, 112, 122]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsResourceStatusClass { +enum HlsResourceStatusClass { Success, Retryable, Permanent, NonRetryable, } -pub fn classify_hls_resource_status(status: StatusCode) -> HlsResourceStatusClass { +fn classify_hls_resource_status(status: StatusCode) -> HlsResourceStatusClass { if status.is_success() { return HlsResourceStatusClass::Success; } @@ -59,6 +67,7 @@ pub enum HlsResourceFetchKind { Segment, Map, Key, + Part, Other, } @@ -68,6 +77,7 @@ impl HlsResourceFetchKind { Self::Segment => "Segment", Self::Map => "Map", Self::Key => "Key", + Self::Part => "Part", Self::Other => "Resource", } } @@ -77,6 +87,7 @@ impl HlsResourceFetchKind { Self::Segment => "segment fetch", Self::Map => "map fetch", Self::Key => "key fetch", + Self::Part => "part fetch", Self::Other => "resource fetch", } } @@ -115,7 +126,7 @@ pub struct HlsOriginResourceFetchTarget { } impl HlsOriginResourceFetchTarget { - pub fn log_context(&self) -> HlsResourceFetchLogContext<'_> { + fn log_context(&self) -> HlsResourceFetchLogContext<'_> { HlsResourceFetchLogContext { kind: self.kind, source: self.source, @@ -143,10 +154,43 @@ pub enum HlsOriginResourceFetchError { InvalidOriginUrl, InvalidByteRange, UnexpectedByteRangeStatus, + ContentCoding(HlsContentCodingFailure), + ContentDecoding(HlsContentDecodingFailure), + CacheObjectLimit { limit: u64 }, CacheCommit(HlsCacheCommitFailure), ProviderUnavailable(HlsBoundAccountAcquireErrorKind), } +/// Non-streaming HTTP content-coding failures that permanently reject one origin object. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsContentCodingFailure { + InvalidHeader, + Unsupported, + EncodedPartialContent, +} + +impl HlsContentCodingFailure { + fn label(&self) -> String { + match self { + Self::InvalidHeader => "ContentCodingError(InvalidHeader)".to_string(), + Self::Unsupported => "ContentCodingError(Unsupported)".to_string(), + Self::EncodedPartialContent => "ContentCodingError(EncodedPartialContent)".to_string(), + } + } +} + +/// Fixed coding class for a retryable failure while streaming a decoded origin body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsContentDecodingFailure { + coding: crate::utils::content_coding::ContentCoding, +} + +impl HlsContentDecodingFailure { + fn from_io_error(error: &ContentDecodingIoError) -> Self { Self { coding: error.coding } } + + fn label(&self) -> String { format!("ContentDecodingError(coding={})", self.coding.as_http_token()) } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsCacheCommitFailure { io_kind: String, @@ -169,9 +213,7 @@ impl HlsCacheCommitFailure { } fn label(&self) -> String { - let raw_os_error = self - .raw_os_error - .map_or_else(|| "none".to_string(), |code| code.to_string()); + let raw_os_error = self.raw_os_error.map_or_else(|| "none".to_string(), |code| code.to_string()); format!( "CacheCommitError(io_kind={} raw_os_error={} storage_full={} message=\"{}\")", self.io_kind, raw_os_error, self.storage_full, self.message @@ -184,12 +226,49 @@ impl HlsCacheCommitFailure { impl HlsOriginResourceFetchError { pub fn cache_commit(err: &io::Error) -> Self { Self::CacheCommit(HlsCacheCommitFailure::from_io_error(err)) } + /// Classifies an error produced while consuming a decoded body into the HLS retry contract. + pub fn cache_body(err: &io::Error) -> Self { + if let Some(error) = content_decoding_error_from_io(err) { + return Self::ContentDecoding(HlsContentDecodingFailure::from_io_error(error)); + } + if err.kind() == io::ErrorKind::TimedOut { + return Self::Timeout; + } + if is_http_body_transport_error(err) { + return Self::Transport(sanitize_sensitive_info(&err.to_string()).to_string()); + } + if let Some(error) = hls_cache_object_limit_from_io(err) { + return Self::CacheObjectLimit { limit: error.limit() }; + } + Self::cache_commit(err) + } + + fn from_body_read_error(err: &io::Error) -> Self { + if let Some(error) = content_decoding_error_from_io(err) { + return Self::ContentDecoding(HlsContentDecodingFailure::from_io_error(error)); + } + if err.kind() == io::ErrorKind::TimedOut { + return Self::Timeout; + } + Self::Transport(sanitize_sensitive_info(&err.to_string()).to_string()) + } + + fn from_content_coding_error(err: ContentCodingError) -> Self { + match err { + ContentCodingError::InvalidHeader => Self::ContentCoding(HlsContentCodingFailure::InvalidHeader), + ContentCodingError::Unsupported(_) => Self::ContentCoding(HlsContentCodingFailure::Unsupported), + ContentCodingError::EncodedPartialContent => { + Self::ContentCoding(HlsContentCodingFailure::EncodedPartialContent) + } + ContentCodingError::PrefixRead(err) => Self::from_body_read_error(&err), + } + } + pub fn retryable_failure(&self) -> bool { matches!( self, - Self::RetryableStatus(_) | Self::Transport(_) | Self::Redirect | Self::Timeout - ) - || matches!(self, Self::CacheCommit(failure) if !failure.storage_full()) + Self::RetryableStatus(_) | Self::Transport(_) | Self::Redirect | Self::Timeout | Self::ContentDecoding(_) + ) || matches!(self, Self::CacheCommit(failure) if !failure.storage_full()) || matches!(self, Self::ProviderUnavailable(kind) if kind.is_retryable_resource_failure()) } @@ -208,6 +287,8 @@ impl HlsOriginResourceFetchError { | Self::InvalidOriginUrl | Self::InvalidByteRange | Self::UnexpectedByteRangeStatus + | Self::ContentCoding(_) + | Self::CacheObjectLimit { .. } ) } @@ -221,6 +302,9 @@ impl HlsOriginResourceFetchError { } Self::Redirect => HlsResourceFetchLogStatus::RedirectError, Self::Timeout => HlsResourceFetchLogStatus::Timeout, + Self::ContentCoding(failure) => HlsResourceFetchLogStatus::ContentCodingError(failure.clone()), + Self::ContentDecoding(failure) => HlsResourceFetchLogStatus::ContentDecodingError(failure.clone()), + Self::CacheObjectLimit { limit } => HlsResourceFetchLogStatus::CacheObjectLimit { limit: *limit }, Self::CacheCommit(failure) => HlsResourceFetchLogStatus::CacheCommitError(failure.clone()), Self::ProviderUnavailable(kind) => HlsResourceFetchLogStatus::ProviderUnavailable(*kind), } @@ -232,67 +316,146 @@ impl HlsOriginResourceFetchError { | Self::NonRetryableStatus(_) | Self::InvalidOriginUrl | Self::InvalidByteRange - | Self::UnexpectedByteRangeStatus => true, + | Self::UnexpectedByteRangeStatus + | Self::ContentCoding(_) + | Self::CacheObjectLimit { .. } => true, Self::ProviderUnavailable(kind) => !kind.is_retryable_resource_failure(), + Self::CacheCommit(failure) => failure.storage_full(), Self::RetryableStatus(_) | Self::Transport(_) | Self::Redirect | Self::Timeout - | Self::CacheCommit(_) => false, + | Self::ContentDecoding(_) => false, } } } -pub type HlsOriginResourceCommitFuture = BoxFuture<'static, Result>; -pub type HlsOriginResourceAttemptPrepareFuture = BoxFuture<'static, Result>; -pub type HlsOriginResourceAttemptCleanupFuture = BoxFuture<'static, ()>; +type HlsOriginResourceResponsePrepareFuture = BoxFuture<'static, Result>; -pub async fn run_hls_origin_resource_retry_loop( +struct HlsOriginResourceRetryRun<'a> { target: HlsOriginResourceFetchTarget, clients: HlsOriginResourceClients, - policy: &SegmentFetchPolicy, - session_log_id: &str, - mut commit: F, -) -> Result -where - T: Send + 'static, - F: FnMut(reqwest::Response, HlsResourceFetchAttempt) -> HlsOriginResourceCommitFuture, -{ - run_hls_origin_resource_retry_loop_with_attempt_prepare( - target, - clients, - policy, - session_log_id, - |_| async { Ok(()) }.boxed(), - |()| async {}.boxed(), - move |response, attempt, ()| commit(response, attempt), - ) - .await + policy: &'a SegmentFetchPolicy, + session_log_id: &'a str, +} + +/// Absolute per-attempt deadline shared by decoder preparation and decoded cache-body consumption. +#[derive(Debug, Clone, Copy)] +pub struct HlsOriginResourceBodyDeadline { + deadline: tokio::time::Instant, + timeout: Duration, +} + +impl HlsOriginResourceBodyDeadline { + fn new(timeout: Duration) -> Self { Self { deadline: tokio::time::Instant::now() + timeout, timeout } } + + /// Returns the remaining body budget, saturating at zero once the deadline is exhausted. + pub fn remaining(self) -> Duration { self.deadline.saturating_duration_since(tokio::time::Instant::now()) } + + /// Returns the absolute deadline used by decoded cache-body consumers. + pub fn deadline(self) -> tokio::time::Instant { self.deadline } + + /// Returns the configured per-attempt body budget for timeout diagnostics. + pub fn timeout(self) -> Duration { self.timeout } } /// Runs the shared HLS origin-resource retry policy with an optional per-attempt guard. /// /// The prepare callback runs after the attempt log and before the HTTP request. If the HTTP -/// request fails before the commit callback takes ownership of the guard, the cleanup callback is -/// invoked by the runner. Once the commit callback receives a guard, it owns its cleanup path. This -/// lets direct passthrough callers return a guard with the response body while cache-commit callers -/// release it after the commit finishes. +/// request or decoder setup fails before the commit callback takes ownership of the guard, the +/// cleanup callback is invoked by the runner. Once the commit callback receives a guard, it owns its +/// cleanup path. Direct client streams return the decoded reader from their commit callback; body +/// errors observed after that handoff deliberately remain outside this retry loop. pub async fn run_hls_origin_resource_retry_loop_with_attempt_prepare( target: HlsOriginResourceFetchTarget, clients: HlsOriginResourceClients, policy: &SegmentFetchPolicy, session_log_id: &str, + prepare_attempt: P, + cleanup_attempt: C, + commit: F, +) -> Result +where + T: Send + 'static, + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, + C: FnMut(G) -> BoxFuture<'static, ()>, + F: FnMut( + DecodedHttpResponse, + HlsResourceFetchAttempt, + HlsOriginResourceBodyDeadline, + G, + ) -> BoxFuture<'static, Result>, +{ + let coding_object_kind = match target.kind { + HlsResourceFetchKind::Segment => HlsOriginContentCodingObjectKind::Segment, + HlsResourceFetchKind::Map => HlsOriginContentCodingObjectKind::Map, + HlsResourceFetchKind::Key => HlsOriginContentCodingObjectKind::Key, + HlsResourceFetchKind::Part => HlsOriginContentCodingObjectKind::Part, + HlsResourceFetchKind::Other => HlsOriginContentCodingObjectKind::Other, + }; + let range_requested = target.headers.contains_key(header::RANGE); + run_hls_origin_resource_raw_retry_core( + HlsOriginResourceRetryRun { target, clients, policy, session_log_id }, + prepare_attempt, + cleanup_attempt, + |response, deadline| { + async move { + let remaining = deadline.remaining(); + if remaining.is_zero() { + return Err(HlsOriginResourceFetchError::Timeout); + } + match tokio::time::timeout_at( + deadline.deadline(), + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly), + ) + .await + { + Ok(result) => { + let decoded = result.map_err(HlsOriginResourceFetchError::from_content_coding_error)?; + if let Some(observation) = decoded.content_coding_observation() { + log_hls_origin_content_coding( + observation, + coding_object_kind, + range_requested, + HlsOriginContentCodingSource::Shared, + ); + } + Ok(decoded) + } + Err(_) => Err(HlsOriginResourceFetchError::Timeout), + } + } + .boxed() + }, + commit, + ) + .await +} + +/// Owns the single logical retry budget shared by decoded cache fetches and direct streams. +async fn run_hls_origin_resource_raw_retry_core( + run: HlsOriginResourceRetryRun<'_>, mut prepare_attempt: P, mut cleanup_attempt: C, + mut prepare_response: D, mut commit: F, ) -> Result where T: Send + 'static, G: Send + 'static, - P: FnMut(HlsResourceFetchAttempt) -> HlsOriginResourceAttemptPrepareFuture, - C: FnMut(G) -> HlsOriginResourceAttemptCleanupFuture, - F: FnMut(reqwest::Response, HlsResourceFetchAttempt, G) -> HlsOriginResourceCommitFuture, + R: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, + C: FnMut(G) -> BoxFuture<'static, ()>, + D: FnMut(reqwest::Response, HlsOriginResourceBodyDeadline) -> HlsOriginResourceResponsePrepareFuture, + F: FnMut( + R, + HlsResourceFetchAttempt, + HlsOriginResourceBodyDeadline, + G, + ) -> BoxFuture<'static, Result>, { + let HlsOriginResourceRetryRun { target, clients, policy, session_log_id } = run; let attempts = policy.retry_delays_ms.len(); for attempt_index in 0..attempts { let attempt = HlsResourceFetchAttempt { attempt_index, attempts }; @@ -305,7 +468,17 @@ where let attempt_started_at = Instant::now(); let result = match prepare_attempt(attempt).await { Ok(guard) => match fetch_hls_origin_resource_response(&target, &clients).await { - Ok(response) => commit(response, attempt, guard).await, + Ok(response) => { + let deadline = + HlsOriginResourceBodyDeadline::new(hls_object_body_deadline(policy.origin_segment_timeout_ms)); + match prepare_response(response, deadline).await { + Ok(response) => commit(response, attempt, deadline, guard).await, + Err(err) => { + cleanup_attempt(guard).await; + Err(err) + } + } + } Err(err) => { cleanup_attempt(guard).await; Err(err) @@ -354,7 +527,7 @@ where Err(HlsOriginResourceFetchError::Timeout) } -pub async fn fetch_hls_origin_resource_response( +async fn fetch_hls_origin_resource_response( target: &HlsOriginResourceFetchTarget, clients: &HlsOriginResourceClients, ) -> Result { @@ -363,17 +536,11 @@ pub async fn fetch_hls_origin_resource_response( fetch_hls_origin_resource_with_manual_redirects(&url, target.headers.clone(), &clients.no_redirect_client) .await? } else { - clients - .client - .get(url) - .headers(target.headers.clone()) - .send() - .await - .map_err(|err| { - HlsOriginResourceFetchError::Transport( - sanitize_sensitive_info(err.to_string().as_str()).to_string(), - ) - })? + let mut headers = target.headers.clone(); + apply_outbound_content_coding_policy(&mut headers, OutboundContentCodingPolicy::Identity); + clients.client.get(url).headers(headers).send().await.map_err(|err| { + HlsOriginResourceFetchError::Transport(sanitize_sensitive_info(err.to_string().as_str()).to_string()) + })? }; classify_hls_origin_resource_response(response, target.byte_range_expectation) } @@ -388,16 +555,11 @@ async fn fetch_hls_origin_resource_with_manual_redirects( let mut remaining_redirects = MAX_MANUAL_REDIRECTS; loop { - let response = client - .get(current_url.clone()) - .headers(current_headers.clone()) - .send() - .await - .map_err(|err| { - HlsOriginResourceFetchError::Transport( - sanitize_sensitive_info(err.to_string().as_str()).to_string(), - ) - })?; + let mut request_headers = current_headers.clone(); + apply_outbound_content_coding_policy(&mut request_headers, OutboundContentCodingPolicy::Identity); + let response = client.get(current_url.clone()).headers(request_headers).send().await.map_err(|err| { + HlsOriginResourceFetchError::Transport(sanitize_sensitive_info(err.to_string().as_str()).to_string()) + })?; if !response.status().is_redirection() { return Ok(response); } @@ -453,38 +615,59 @@ pub fn build_hls_origin_resource_headers( provider_session_headers: &HeaderMap, byte_range: Option, ) -> Result { - let mut headers = source_headers.clone(); - scrub_hls_origin_headers(&mut headers, None); - force_identity_without_range(&mut headers); - append_hls_provider_session_headers(&mut headers, provider_session_headers); - if let Some(byte_range) = byte_range { - let end = byte_range - .offset - .checked_add(byte_range.length) - .and_then(|end_exclusive| end_exclusive.checked_sub(1)) - .ok_or(HlsOriginResourceFetchError::InvalidByteRange)?; - let range_value = format!("bytes={}-{}", byte_range.offset, end); - let range_value = HeaderValue::from_str(&range_value) - .map_err(|_| HlsOriginResourceFetchError::InvalidByteRange)?; - headers.insert(header::RANGE, range_value); - } - Ok(headers) + build_hls_origin_resource_headers_with_range( + source_headers, + provider_session_headers, + byte_range.map_or(HlsOriginRangeHeader::None, HlsOriginRangeHeader::Parsed), + ) } pub fn build_hls_origin_resource_headers_with_client_range( source_headers: &HeaderMap, provider_session_headers: &HeaderMap, client_range: Option, -) -> HeaderMap { +) -> Result { + build_hls_origin_resource_headers_with_range( + source_headers, + provider_session_headers, + client_range.map_or(HlsOriginRangeHeader::None, HlsOriginRangeHeader::Forwarded), + ) +} + +/// The sole controlled `Range` value inserted after origin-header scrubbing. +enum HlsOriginRangeHeader { + None, + Parsed(ParsedByteRange), + Forwarded(HeaderValue), +} + +fn build_hls_origin_resource_headers_with_range( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + range: HlsOriginRangeHeader, +) -> Result { let mut headers = source_headers.clone(); scrub_hls_origin_headers(&mut headers, None); - headers.remove(header::RANGE); - headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("identity")); append_hls_provider_session_headers(&mut headers, provider_session_headers); - if let Some(range) = client_range { - headers.insert(header::RANGE, range); + force_identity_without_range(&mut headers); + match range { + HlsOriginRangeHeader::None => {} + HlsOriginRangeHeader::Parsed(byte_range) => { + let end = byte_range + .offset + .checked_add(byte_range.length) + .and_then(|end_exclusive| end_exclusive.checked_sub(1)) + .ok_or(HlsOriginResourceFetchError::InvalidByteRange)?; + let range_value = format!("bytes={}-{}", byte_range.offset, end); + let range_value = + HeaderValue::from_str(&range_value).map_err(|_| HlsOriginResourceFetchError::InvalidByteRange)?; + headers.insert(header::RANGE, range_value); + } + HlsOriginRangeHeader::Forwarded(range) => { + headers.insert(header::RANGE, range); + } } - headers + Ok(headers) } fn same_origin(lhs: &Url, rhs: &Url) -> bool { @@ -498,25 +681,28 @@ fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HeaderMap) { } #[derive(Debug, Clone, Copy)] -pub struct HlsResourceFetchLogContext<'a> { - pub kind: HlsResourceFetchKind, - pub source: HlsResourceFetchSource, - pub object_id: &'a str, - pub origin_url: Option<&'a str>, +pub(super) struct HlsResourceFetchLogContext<'a> { + pub(super) kind: HlsResourceFetchKind, + pub(super) source: HlsResourceFetchSource, + pub(super) object_id: &'a str, + pub(super) origin_url: Option<&'a str>, } #[derive(Debug, Clone)] -pub enum HlsResourceFetchLogStatus { +enum HlsResourceFetchLogStatus { Http(StatusCode), Timeout, TransportError, RedirectError, + ContentCodingError(HlsContentCodingFailure), + ContentDecodingError(HlsContentDecodingFailure), + CacheObjectLimit { limit: u64 }, CacheCommitError(HlsCacheCommitFailure), ProviderUnavailable(HlsBoundAccountAcquireErrorKind), } impl HlsResourceFetchLogStatus { - pub fn label(self) -> String { + fn label(self) -> String { match self { Self::Http(status) => { let reason = status.canonical_reason().unwrap_or("Unknown"); @@ -525,6 +711,9 @@ impl HlsResourceFetchLogStatus { Self::Timeout => "Timeout".to_string(), Self::TransportError => "TransportError".to_string(), Self::RedirectError => "RedirectError".to_string(), + Self::ContentCodingError(failure) => failure.label(), + Self::ContentDecodingError(failure) => failure.label(), + Self::CacheObjectLimit { limit } => format!("CacheObjectLimit(limit={limit})"), Self::CacheCommitError(failure) => failure.label(), Self::ProviderUnavailable(kind) => format!("ProviderUnavailable({})", kind.as_log_label()), } @@ -537,11 +726,28 @@ pub struct HlsResourceFetchAttempt { pub attempts: usize, } +impl HlsResourceFetchAttempt { + /// Returns the one-based attempt number used in human-readable diagnostics. + const fn display_number(self) -> usize { self.attempt_index + 1 } +} + +#[cfg(test)] +thread_local! { + static BODY_FAILURE_LOG_ATTEMPTS: std::cell::RefCell> = const { + std::cell::RefCell::new(Vec::new()) + }; +} + +#[cfg(test)] +pub(super) fn take_body_failure_log_attempts() -> Vec<(usize, usize)> { + BODY_FAILURE_LOG_ATTEMPTS.with(|attempts| std::mem::take(&mut *attempts.borrow_mut())) +} + fn safe_resource_fetch_session_id(session: &str) -> String { safe_proxy_session_id(&ProxySessionId(session.to_string())) } -pub fn log_hls_resource_attempt_started( +fn log_hls_resource_attempt_started( session: &str, context: HlsResourceFetchLogContext<'_>, attempt: HlsResourceFetchAttempt, @@ -551,7 +757,7 @@ pub fn log_hls_resource_attempt_started( "{} '{}' attempting URL attempt {} of {}: session={} source={} {}", context.kind.label(), context.object_id, - attempt.attempt_index, + attempt.display_number(), attempt.attempts, safe_resource_fetch_session_id(session), context.source.as_log_value(), @@ -562,7 +768,7 @@ pub fn log_hls_resource_attempt_started( "{} '{}' attempting URL attempt {} of {}: session={} source={}", context.kind.label(), context.object_id, - attempt.attempt_index, + attempt.display_number(), attempt.attempts, safe_resource_fetch_session_id(session), context.source.as_log_value() @@ -570,11 +776,7 @@ pub fn log_hls_resource_attempt_started( } } -pub fn log_hls_resource_attempt_succeeded( - session: &str, - context: HlsResourceFetchLogContext<'_>, - elapsed: Duration, -) { +fn log_hls_resource_attempt_succeeded(session: &str, context: HlsResourceFetchLogContext<'_>, elapsed: Duration) { debug!( "{} '{}' success: session={} source={} {} took {:.3}s", context.kind.label(), @@ -586,7 +788,7 @@ pub fn log_hls_resource_attempt_succeeded( ); } -pub fn log_hls_resource_retry_scheduled( +fn log_hls_resource_retry_scheduled( session: &str, context: HlsResourceFetchLogContext<'_>, attempt: HlsResourceFetchAttempt, @@ -600,13 +802,13 @@ pub fn log_hls_resource_retry_scheduled( safe_resource_fetch_session_id(session), context.source.as_log_value(), status.label(), - attempt.attempt_index, + attempt.display_number(), attempt.attempts, next_delay_ms ); } -pub fn log_hls_resource_fetch_failed( +fn log_hls_resource_fetch_failed( session: &str, context: HlsResourceFetchLogContext<'_>, attempt: HlsResourceFetchAttempt, @@ -619,12 +821,12 @@ pub fn log_hls_resource_fetch_failed( safe_resource_fetch_session_id(session), context.source.as_log_value(), status.label(), - attempt.attempt_index, + attempt.display_number(), attempt.attempts ); } -pub fn log_hls_resource_timeout( +fn log_hls_resource_timeout( session: &str, context: HlsResourceFetchLogContext<'_>, attempt: HlsResourceFetchAttempt, @@ -636,12 +838,31 @@ pub fn log_hls_resource_timeout( context.source.as_log_value(), context.kind.label(), context.object_id, - attempt.attempt_index, + attempt.display_number(), attempt.attempts, deadline_ms ); } +/// Logs one sanitized terminal diagnostic for an origin-body failure after response handoff. +pub(super) fn log_hls_resource_body_failure( + session: &str, + context: HlsResourceFetchLogContext<'_>, + attempt: HlsResourceFetchAttempt, + error: &io::Error, + deadline_ms: u128, +) { + #[cfg(test)] + BODY_FAILURE_LOG_ATTEMPTS.with(|attempts| attempts.borrow_mut().push((attempt.attempt_index, attempt.attempts))); + + let failure = HlsOriginResourceFetchError::from_body_read_error(error); + if matches!(&failure, HlsOriginResourceFetchError::Timeout) { + log_hls_resource_timeout(session, context, attempt, deadline_ms); + } else { + log_hls_resource_fetch_failed(session, context, attempt, failure.log_status()); + } +} + pub fn retry_after_secs_from_ms(retry_after_ms: u64) -> u64 { retry_after_ms.saturating_add(999).saturating_div(1_000).max(1) } @@ -649,12 +870,66 @@ pub fn retry_after_secs_from_ms(retry_after_ms: u64) -> u64 { #[cfg(test)] mod tests { use super::{ - build_hls_origin_resource_headers_with_client_range, HlsCacheCommitFailure, HlsOriginResourceFetchError, - HlsResourceFetchLogStatus, + build_hls_origin_resource_headers, build_hls_origin_resource_headers_with_client_range, + fetch_hls_origin_resource_response, run_hls_origin_resource_retry_loop_with_attempt_prepare, + HlsCacheCommitFailure, HlsContentCodingFailure, HlsOriginByteRangeExpectation, HlsOriginResourceBodyDeadline, + HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, + HlsResourceFetchAttempt, HlsResourceFetchLogStatus, HlsResourceFetchSource, + }; + use crate::{ + api::model::{HlsBoundAccountAcquireErrorKind, SegmentFetchPolicy}, + processing::parser::hls::origin_manifest::ParsedByteRange, + utils::content_coding::{ContentCoding, ContentCodingError, ContentDecodingIoError}, }; - use crate::api::model::HlsBoundAccountAcquireErrorKind; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; - use std::io; + use futures::FutureExt; + use reqwest::{redirect::Policy, Client}; + use std::{ + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + }; + + async fn read_request(stream: &mut TcpStream) -> String { + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let read = stream.read(&mut buffer).await.expect("read request"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + } + String::from_utf8(request).expect("request should be HTTP text") + } + + fn fetch_target(origin_url: String) -> HlsOriginResourceFetchTarget { + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + HlsOriginResourceFetchTarget { + kind: HlsResourceFetchKind::Segment, + source: HlsResourceFetchSource::Normal, + object_id: "segment".to_string(), + origin_url, + headers, + byte_range_expectation: HlsOriginByteRangeExpectation::FullObject, + } + } + + fn fetch_clients(use_manual_redirects: bool) -> HlsOriginResourceClients { + HlsOriginResourceClients { + client: Client::builder().build().expect("client"), + no_redirect_client: Client::builder().redirect(Policy::none()).build().expect("no redirect client"), + use_manual_redirects, + } + } #[test] fn formats_http_fetch_log_status_with_reason() { @@ -671,14 +946,302 @@ mod tests { assert_eq!(HlsResourceFetchLogStatus::RedirectError.label(), "RedirectError"); } + #[test] + fn resource_fetch_attempt_display_number_is_one_based_without_changing_raw_index() { + let first = HlsResourceFetchAttempt { attempt_index: 0, attempts: 3 }; + let last = HlsResourceFetchAttempt { attempt_index: 2, attempts: 3 }; + + assert_eq!(first.display_number(), 1); + assert_eq!(last.display_number(), 3); + assert_eq!(first.attempt_index, 0); + assert_eq!(last.attempt_index, 2); + assert_eq!(first.attempts, 3); + assert_eq!(last.attempts, 3); + } + + #[test] + fn exhausted_body_deadline_saturates_at_zero() { + let deadline = HlsOriginResourceBodyDeadline::new(Duration::ZERO); + + assert_eq!(deadline.remaining(), Duration::ZERO); + } + + #[test] + fn unsupported_content_coding_is_a_permanent_object_failure() { + let error = HlsOriginResourceFetchError::from_content_coding_error(ContentCodingError::Unsupported( + "compress".to_string(), + )); + + assert!(error.object_failure_is_permanent()); + assert!(!error.retryable_failure()); + assert!(error.aborts_without_retry()); + assert_eq!(error.log_status().label(), "ContentCodingError(Unsupported)"); + } + + #[test] + fn streaming_decoder_failure_is_retryable_and_not_a_cache_commit_failure() { + let error = io::Error::new(io::ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Gzip }); + + let error = HlsOriginResourceFetchError::cache_body(&error); + + assert!(matches!(&error, HlsOriginResourceFetchError::ContentDecoding(_))); + assert!(error.retryable_failure()); + assert!(!error.object_failure_is_permanent()); + } + + #[test] + fn direct_body_failure_classification_distinguishes_decoder_transport_and_timeout() { + let decoder_error = + io::Error::new(io::ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Gzip }); + let decoder_failure = HlsOriginResourceFetchError::from_body_read_error(&decoder_error); + assert!(matches!(&decoder_failure, HlsOriginResourceFetchError::ContentDecoding(_))); + assert_eq!(decoder_failure.log_status().label(), "ContentDecodingError(coding=gzip)"); + + let transport_error = io::Error::new(io::ErrorKind::ConnectionReset, "origin body reset"); + let transport_failure = HlsOriginResourceFetchError::from_body_read_error(&transport_error); + assert!(matches!(&transport_failure, HlsOriginResourceFetchError::Transport(_))); + assert_eq!(transport_failure.log_status().label(), "TransportError"); + + let timeout_error = io::Error::new(io::ErrorKind::TimedOut, "origin body timed out"); + let timeout_failure = HlsOriginResourceFetchError::from_body_read_error(&timeout_error); + assert!(matches!(&timeout_failure, HlsOriginResourceFetchError::Timeout)); + assert_eq!(timeout_failure.log_status().label(), "Timeout"); + } + + #[tokio::test] + async fn final_automatic_origin_request_overrides_accept_encoding_with_identity() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind local server"); + let address = listener.local_addr().expect("local address"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept request"); + let request = read_request(&mut stream).await; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("write response"); + request + }); + + let target = fetch_target(format!("http://{address}/segment.ts")); + let response = + fetch_hls_origin_resource_response(&target, &fetch_clients(false)).await.expect("origin response"); + assert_eq!(response.status(), StatusCode::OK); + + let request = server.await.expect("server task").to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity\r\n")); + assert!(!request.contains("accept-encoding: gzip\r\n")); + } + + #[tokio::test] + async fn every_manual_redirect_request_enforces_accept_encoding_identity() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind local server"); + let address = listener.local_addr().expect("local address"); + let server = tokio::spawn(async move { + let mut requests = Vec::new(); + for response in [ + "HTTP/1.1 302 Found\r\nLocation: /next.ts\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ] { + let (mut stream, _) = listener.accept().await.expect("accept request"); + requests.push(read_request(&mut stream).await); + stream.write_all(response.as_bytes()).await.expect("write response"); + } + requests + }); + + let target = fetch_target(format!("http://{address}/segment.ts")); + let response = fetch_hls_origin_resource_response(&target, &fetch_clients(true)) + .await + .expect("redirected origin response"); + assert_eq!(response.status(), StatusCode::OK); + + let requests = server.await.expect("server task"); + assert_eq!(requests.len(), 2); + for request in requests { + let request = request.to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity\r\n")); + assert!(!request.contains("accept-encoding: gzip\r\n")); + } + } + + #[tokio::test] + async fn automatic_cross_origin_redirect_preserves_identity_and_range_but_strips_provider_cookie() { + let destination_listener = TcpListener::bind("127.0.0.1:0").await.expect("bind redirect destination"); + let destination_address = destination_listener.local_addr().expect("redirect destination address"); + let destination = tokio::spawn(async move { + let (mut stream, _) = destination_listener.accept().await.expect("accept redirected request"); + let request = read_request(&mut stream).await; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("write redirected response"); + request + }); + + let source_listener = TcpListener::bind("127.0.0.1:0").await.expect("bind redirect source"); + let source_address = source_listener.local_addr().expect("redirect source address"); + let source = tokio::spawn(async move { + let (mut stream, _) = source_listener.accept().await.expect("accept initial request"); + let request = read_request(&mut stream).await; + let response = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{destination_address}/final.ts\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + stream.write_all(response.as_bytes()).await.expect("write redirect response"); + request + }); + + let mut provider_headers = HeaderMap::new(); + provider_headers.insert(header::COOKIE, HeaderValue::from_static("sid=provider")); + let headers = build_hls_origin_resource_headers_with_client_range( + &HeaderMap::new(), + &provider_headers, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .expect("forwarded range headers"); + let mut target = fetch_target(format!("http://{source_address}/segment.ts")); + target.headers = headers; + + let response = fetch_hls_origin_resource_response(&target, &fetch_clients(false)) + .await + .expect("automatic redirect response"); + assert_eq!(response.status(), StatusCode::OK); + + let initial_request = source.await.expect("initial request task").to_ascii_lowercase(); + let redirected_request = destination.await.expect("redirected request task").to_ascii_lowercase(); + for request in [&initial_request, &redirected_request] { + assert!(request.contains("accept-encoding: identity\r\n")); + assert!(request.contains("range: bytes=2-5\r\n")); + assert!(!request.contains("authorization:")); + assert!(!request.contains("proxy-authorization:")); + } + assert!(initial_request.contains("cookie: sid=provider\r\n")); + assert!(!redirected_request.contains("cookie:")); + } + + #[tokio::test] + async fn full_object_partial_response_is_rejected_before_decoder_and_cleans_attempt_guard() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind local server"); + let address = listener.local_addr().expect("local address"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept request"); + let request = read_request(&mut stream).await; + stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\nContent-Range: bytes 0-2/10\r\nContent-Length: 3\r\nConnection: close\r\n\r\nbad", + ) + .await + .expect("write response"); + request + }); + let target = fetch_target(format!("http://{address}/segment.ts")); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let commit_count = Arc::new(AtomicUsize::new(0)); + let cleanup_count_for_callback = Arc::clone(&cleanup_count); + let commit_count_for_callback = Arc::clone(&commit_count); + + let result = run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + fetch_clients(false), + &policy, + "session", + |_| async { Ok(()) }.boxed(), + move |()| { + let cleanup_count = Arc::clone(&cleanup_count_for_callback); + async move { + cleanup_count.fetch_add(1, Ordering::Relaxed); + } + .boxed() + }, + move |_response, _attempt, _deadline, ()| { + let commit_count = Arc::clone(&commit_count_for_callback); + async move { + commit_count.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + .boxed() + }, + ) + .await; + + assert!(matches!(result, Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus))); + assert_eq!(cleanup_count.load(Ordering::Relaxed), 1); + assert_eq!(commit_count.load(Ordering::Relaxed), 0); + let request = server.await.expect("server task").to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity\r\n")); + } + + #[tokio::test] + async fn range_accepted_encoded_partial_response_is_rejected_by_content_coding_policy() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind local server"); + let address = listener.local_addr().expect("local address"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept request"); + let request = read_request(&mut stream).await; + stream + .write_all( + b"HTTP/1.1 206 Partial Content\r\nContent-Encoding: gzip\r\nContent-Range: bytes 0-2/10\r\nContent-Length: 3\r\nConnection: close\r\n\r\nbad", + ) + .await + .expect("write response"); + request + }); + let mut target = fetch_target(format!("http://{address}/segment.ts")); + target.byte_range_expectation = HlsOriginByteRangeExpectation::PartialContent; + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let commit_count = Arc::new(AtomicUsize::new(0)); + let cleanup_count_for_callback = Arc::clone(&cleanup_count); + let commit_count_for_callback = Arc::clone(&commit_count); + + let result = run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + fetch_clients(false), + &policy, + "session", + |_| async { Ok(()) }.boxed(), + move |()| { + let cleanup_count = Arc::clone(&cleanup_count_for_callback); + async move { + cleanup_count.fetch_add(1, Ordering::Relaxed); + } + .boxed() + }, + move |_response, _attempt, _deadline, ()| { + let commit_count = Arc::clone(&commit_count_for_callback); + async move { + commit_count.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + .boxed() + }, + ) + .await; + + assert!(matches!( + result, + Err(HlsOriginResourceFetchError::ContentCoding(HlsContentCodingFailure::EncodedPartialContent)) + )); + assert_eq!(cleanup_count.load(Ordering::Relaxed), 1); + assert_eq!(commit_count.load(Ordering::Relaxed), 0); + let request = server.await.expect("server task").to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity\r\n")); + } + #[test] fn cache_commit_failure_marks_enospc_as_storage_full() { let failure = HlsCacheCommitFailure::from_io_error(&io::Error::from_raw_os_error(28)); assert!(failure.storage_full()); - assert!(HlsResourceFetchLogStatus::CacheCommitError(failure) - .label() - .contains("storage_full=true")); + assert!(HlsResourceFetchLogStatus::CacheCommitError(failure).label().contains("storage_full=true")); } #[test] @@ -686,6 +1249,51 @@ mod tests { let err = HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28)); assert!(!err.retryable_failure()); + assert!(err.aborts_without_retry()); + } + + #[tokio::test] + async fn storage_full_cache_commit_aborts_after_one_origin_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind local server"); + let address = listener.local_addr().expect("local address"); + let request_count = Arc::new(AtomicUsize::new(0)); + let server_request_count = Arc::clone(&request_count); + let server = tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + read_request(&mut stream).await; + server_request_count.fetch_add(1, Ordering::Relaxed); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .expect("write response"); + } + }); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + + let result = run_hls_origin_resource_retry_loop_with_attempt_prepare( + fetch_target(format!("http://{address}/segment.ts")), + fetch_clients(false), + &policy, + "session", + |_| async { Ok(()) }.boxed(), + |()| async {}.boxed(), + |_response, _attempt, _deadline, ()| { + async { Err::<(), _>(HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28))) } + .boxed() + }, + ) + .await; + + server.abort(); + assert!(matches!(result, Err(HlsOriginResourceFetchError::CacheCommit(failure)) if failure.storage_full())); + assert_eq!(request_count.load(Ordering::Relaxed), 1); } #[test] @@ -713,11 +1321,48 @@ mod tests { let mut provider_headers = HeaderMap::new(); provider_headers.insert(header::COOKIE, HeaderValue::from_static("sid=provider")); - let headers = - build_hls_origin_resource_headers_with_client_range(&client_headers, &provider_headers, None); + let headers = build_hls_origin_resource_headers_with_client_range(&client_headers, &provider_headers, None) + .expect("origin headers"); assert_eq!(headers.get(header::COOKIE).expect("cookie"), "sid=provider"); assert!(!headers.contains_key(header::RANGE)); assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); } + + #[test] + fn origin_resource_header_wrappers_share_scrubbing_identity_and_range_policy() { + let mut client_headers = HeaderMap::new(); + client_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer client")); + client_headers.insert(header::COOKIE, HeaderValue::from_static("client=secret")); + client_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + client_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + client_headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + let mut provider_headers = HeaderMap::new(); + provider_headers.insert(header::COOKIE, HeaderValue::from_static("sid=provider")); + + let without_range = + build_hls_origin_resource_headers(&client_headers, &provider_headers, None).expect("full object headers"); + let parsed = build_hls_origin_resource_headers( + &client_headers, + &provider_headers, + Some(ParsedByteRange { offset: 2, length: 4 }), + ) + .expect("parsed range headers"); + let forwarded = build_hls_origin_resource_headers_with_client_range( + &client_headers, + &provider_headers, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .expect("forwarded range headers"); + + assert_eq!(parsed, forwarded); + for headers in [&without_range, &parsed, &forwarded] { + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert_eq!(headers.get(header::COOKIE).expect("provider cookie"), "sid=provider"); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("identity"), "identity"); + assert_eq!(headers.get(header::ACCEPT_LANGUAGE).expect("safe input header"), "de"); + } + assert!(!without_range.contains_key(header::RANGE)); + assert_eq!(parsed.get(header::RANGE).expect("parsed range"), "bytes=2-5"); + } } diff --git a/backend/src/api/model/hls_cache/response.rs b/backend/src/api/model/hls_cache/response.rs index 001e83ef1..cba90ed57 100644 --- a/backend/src/api/model/hls_cache/response.rs +++ b/backend/src/api/model/hls_cache/response.rs @@ -2,11 +2,10 @@ use super::{ hls_client_body_send_deadline, refresh_hls_client_body_send_deadline, safe_hls_access_lease_id, - safe_proxy_session_id, CacheAccessState, - HlsAccessLeaseId, HlsCacheMetrics, HlsMapFile, HlsProxyManager, HlsRepairRenderedObjectId, HlsSegmentCache, - HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, - HlsSessionHandle, MapCacheKey, MapCacheStatus, ProxyMapId, SegmentCacheKey, SegmentCacheStatus, - TransientObjectCacheKey, TransientResourceFile, TransientResourceKind, + safe_proxy_session_id, CacheAccessState, HlsAccessLeaseId, HlsCacheMetrics, HlsMapFile, HlsProxyManager, + HlsRepairRenderedObjectId, HlsSegmentCache, HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, HlsSessionHandle, MapCacheKey, MapCacheStatus, ProxyMapId, SegmentCacheKey, + SegmentCacheStatus, TransientObjectCacheKey, TransientResourceFile, TransientResourceKind, }; use crate::api::{api_utils::mark_response_as_uncompressed, model::StreamMeterHandle}; use arc_swap::ArcSwapOption; @@ -19,8 +18,8 @@ use bytes::Bytes; use futures::Stream; use log::debug; use std::{ - io, future::Future, + io, pin::Pin, sync::{ atomic::{AtomicU64, Ordering}, @@ -29,8 +28,10 @@ use std::{ task::{Context, Poll}, time::{Duration, Instant}, }; -use tokio::io::AsyncReadExt; -use tokio::time::{sleep, Sleep}; +use tokio::{ + io::AsyncReadExt, + time::{sleep, Sleep}, +}; use tokio_util::io::ReaderStream; const ACCEPT_RANGES_VALUE: &str = "bytes"; @@ -123,9 +124,7 @@ impl HlsCacheResponseContext { } } - pub fn set_qos_meter(&self, qos_meter: Option>) { - self.qos_meter.store(qos_meter); - } + pub fn set_qos_meter(&self, qos_meter: Option>) { self.qos_meter.store(qos_meter); } } #[derive(Clone)] @@ -196,15 +195,15 @@ pub async fn serve_hls_segment_cache_outcome( context: &HlsCacheResponseContext, ) -> HlsResourceServeOutcome { match lookup_segment_cache_object(&session, &segment_file, &context.hls_access_lease_id).await { - CacheObjectLookup::Ready(object) => { - HlsResourceServeOutcome::Ready(serve_cache_object( + CacheObjectLookup::Ready(object) => HlsResourceServeOutcome::Ready( + serve_cache_object( segment_cache, object, range_header, CacheObjectServeContext::from_response_context(context), ) - .await) - } + .await, + ), CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), } } @@ -232,15 +231,15 @@ pub async fn serve_hls_map_cache_outcome( context: &HlsCacheResponseContext, ) -> HlsResourceServeOutcome { match lookup_map_cache_object(&session, &map_file, &context.hls_access_lease_id).await { - CacheObjectLookup::Ready(object) => { - HlsResourceServeOutcome::Ready(serve_cache_object( + CacheObjectLookup::Ready(object) => HlsResourceServeOutcome::Ready( + serve_cache_object( segment_cache, object, range_header, CacheObjectServeContext::from_response_context(context), ) - .await) - } + .await, + ), CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), } } @@ -270,15 +269,15 @@ pub async fn serve_hls_transient_object_cache_outcome( match lookup_transient_object_cache_object(&session, &resource_file, &context.hls_access_lease_id, context.now_ms) .await { - CacheObjectLookup::Ready(object) => { - HlsResourceServeOutcome::Ready(serve_cache_object( + CacheObjectLookup::Ready(object) => HlsResourceServeOutcome::Ready( + serve_cache_object( segment_cache, object, range_header, CacheObjectServeContext::from_response_context(context), ) - .await) - } + .await, + ), CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), } } @@ -305,10 +304,8 @@ where { let guard = CacheReadGuard::new(Arc::clone(&object.access), context.now_ms); if let Some(repair_context) = object.repair_context.clone() { - if let Err(err) = context - .segment_repair - .repair_ready_cache_hit(&segment_cache, &object.key, repair_context) - .await + if let Err(err) = + context.segment_repair.repair_ready_cache_hit(&segment_cache, &object.key, repair_context).await { debug!( "HLS segment repair skipped for ready cache hit: session={} resource={} error={err}", @@ -425,9 +422,7 @@ async fn lookup_segment_cache_object( } match entry.status { SegmentCacheStatus::Ready { .. } => {} - SegmentCacheStatus::Fetching { .. } - | SegmentCacheStatus::Queued { .. } - | SegmentCacheStatus::Discovered => { + SegmentCacheStatus::Fetching { .. } | SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Discovered => { return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: NOT_READY_RETRY_AFTER_MS, }); @@ -446,13 +441,13 @@ async fn lookup_segment_cache_object( key: entry.cache_key.clone(), access: Arc::clone(&entry.access), content_type: entry.content_type.clone(), - log_context: CacheObjectLogContext { - lease: safe_hls_access_lease_id(hls_access_lease_id), - session: safe_proxy_session_id(&session.proxy_session_id), - resource_id: format!("{:06}", segment_file.proxy_seq), - object_kind: "Segment", - body_source: "normal", - }, + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + session: safe_proxy_session_id(&session.proxy_session_id), + resource_id: format!("{:06}", segment_file.proxy_seq), + object_kind: "Segment", + body_source: "normal", + }, repair_context: Some(HlsSegmentRepairObjectContext { source: HlsSegmentRepairSource::Normal, proxy_session_id: session.proxy_session_id.clone(), @@ -492,9 +487,7 @@ async fn lookup_map_cache_object( } match entry.status { MapCacheStatus::Ready { .. } => {} - MapCacheStatus::Fetching { .. } - | MapCacheStatus::Queued { .. } - | MapCacheStatus::Discovered => { + MapCacheStatus::Fetching { .. } | MapCacheStatus::Queued { .. } | MapCacheStatus::Discovered => { return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: NOT_READY_RETRY_AFTER_MS, }); @@ -620,11 +613,7 @@ fn service_unavailable_not_ready_response(retry_after_ms: u64) -> Response let mut response = Response::new(Body::empty()); *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; let headers = response.headers_mut(); - insert_header_value( - headers, - header::RETRY_AFTER, - &super::retry_after_secs_from_ms(retry_after_ms).to_string(), - ); + insert_header_value(headers, header::RETRY_AFTER, &super::retry_after_secs_from_ms(retry_after_ms).to_string()); headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); mark_response_as_uncompressed(&mut response); response @@ -777,10 +766,7 @@ impl Stream for ActiveReaderStream { if self.send_deadline.as_mut().poll(cx).is_ready() { self.finished = true; self.log_completed("timeout"); - return Poll::Ready(Some(Err(io::Error::new( - io::ErrorKind::TimedOut, - "hls client body send timed out", - )))); + return Poll::Ready(Some(Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")))); } match self.inner.as_mut().poll_next(cx) { Poll::Ready(Some(Ok(chunk))) => { @@ -849,9 +835,13 @@ fn transient_body_object_kind(resource_kind: Option, exte match resource_kind { Some(TransientResourceKind::Key) => "Key", Some(TransientResourceKind::Map) => "Map", - Some(TransientResourceKind::Segment | TransientResourceKind::Other) => "Segment", + Some(TransientResourceKind::Segment | TransientResourceKind::Part | TransientResourceKind::Other) => "Segment", None => { - if extension.eq_ignore_ascii_case("key") { "Key" } else { "Segment" } + if extension.eq_ignore_ascii_case("key") { + "Key" + } else { + "Segment" + } } } } @@ -880,9 +870,9 @@ mod tests { use axum::http::{header, HeaderValue, StatusCode}; use bytes::Bytes; use http_body_util::BodyExt; + use shared::model::HlsSegmentRepairMode; use std::{sync::Arc, time::Duration}; use tokio::sync::RwLock; - use shared::model::HlsSegmentRepairMode; fn header(value: &str) -> HeaderValue { HeaderValue::from_str(value).expect("valid header") } @@ -941,6 +931,7 @@ mod tests { assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Key), "bin"), "Key"); assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Map), "bin"), "Map"); assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Segment), "key"), "Segment"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Part), "m4s"), "Segment"); assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Other), "bin"), "Segment"); } diff --git a/backend/src/api/model/hls_cache/segment_fetcher.rs b/backend/src/api/model/hls_cache/segment_fetcher.rs index 85570c547..5ecb74a92 100644 --- a/backend/src/api/model/hls_cache/segment_fetcher.rs +++ b/backend/src/api/model/hls_cache/segment_fetcher.rs @@ -2,30 +2,30 @@ use super::{ begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, classify_hls_backpressure, - finish_hls_origin_account_io, hls_object_body_deadline, - run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, - HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, - HlsBoundAccountAcquireErrorKind, HlsCacheMetrics, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, - HlsOriginIoContext, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, - HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentFailureObject, - HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, - HlsSegmentRepairSource, HlsSessionHandle, OriginSegmentFetchRef, SegmentCacheKey, SegmentCacheStatus, - SegmentFetchPriority, + finish_hls_origin_account_io, hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, + CachedSegmentMetadata, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeaseStore, + HlsBackpressureState, HlsBoundAccountAcquireErrorKind, HlsCacheMetrics, HlsOriginAccountIoLeaseGuard, + HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, HlsResourceFetchKind, + HlsResourceFetchSource, HlsSegmentCache, HlsSegmentFailureObject, HlsSegmentFailureTransition, HlsSegmentFile, + HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, HlsSessionHandle, + OriginSegmentFetchRef, SegmentCacheKey, SegmentCacheStatus, SegmentFetchPriority, +}; +use crate::{ + model::HlsCacheConfig, processing::parser::hls::origin_manifest::ParsedByteRange, + utils::content_coding::DecodedHttpResponse, }; -use crate::model::HlsCacheConfig; -use crate::processing::parser::hls::origin_manifest::ParsedByteRange; -use shared::model::{HlsSegmentRepairMode, HlsStripMode}; use arc_swap::ArcSwap; use axum::http::HeaderMap; -use futures::{FutureExt, TryStreamExt}; +use futures::FutureExt; use log::{debug, warn}; use reqwest::Client; -use std::{fmt, io, sync::Arc, time::Duration}; +use shared::model::{HlsSegmentRepairMode, HlsStripMode}; +use std::{fmt, sync::Arc, time::Duration}; use tokio::{ sync::{OwnedSemaphorePermit, RwLock, Semaphore}, time::timeout, }; -use tokio_util::io::StreamReader; const DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES: usize = 64; const DEFAULT_MAX_SESSION_SEGMENT_FETCHES: usize = 2; @@ -483,14 +483,12 @@ impl HlsSegmentWorkerPool { threshold ); invalidate_queued_origin_work = true; - response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::SegmentTemporaryFailureThreshold { - failures, - threshold, - }); - SegmentCacheStatus::FailedPermanent { - failed_at_ms: finished_at_ms, - status: None, - } + response_flag_reason = + Some(HlsAccessLeaseChannelUnavailableReason::SegmentTemporaryFailureThreshold { + failures, + threshold, + }); + SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None } } } } else { @@ -508,10 +506,8 @@ impl HlsSegmentWorkerPool { if invalidate_queued_origin_work { session.invalidate_queued_origin_work(); if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = SegmentCacheStatus::FailedPermanent { - failed_at_ms: finished_at_ms, - status: None, - }; + entry.status = + SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None }; } } } @@ -553,10 +549,7 @@ impl HlsSegmentWorkerPool { reason: HlsAccessLeaseChannelUnavailableReason, ) -> usize { let proxy_session_id = session.read().await.proxy_session_id.clone(); - self.access_leases - .write() - .await - .mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) + self.access_leases.write().await.mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) } } @@ -700,7 +693,6 @@ async fn fetch_segment_with_retries_into_cache( let session_log_id = context.session.read().await.proxy_session_id.0.clone(); let context = context.clone(); let snapshot = snapshot.clone(); - let commit_policy = policy.clone(); let policy_for_prepare = policy.clone(); let prepare_context = context.clone(); let cleanup_context = context.clone(); @@ -721,12 +713,12 @@ async fn fetch_segment_with_retries_into_cache( } .boxed() }, - move |response, _attempt, guard| { + move |response, _attempt, body_deadline, guard| { let context = context.clone(); let snapshot = snapshot.clone(); - let policy = commit_policy.clone(); async move { - let commit_result = commit_segment_response_into_cache(&context, &snapshot, &policy, response).await; + let commit_result = + commit_segment_response_into_cache(&context, &snapshot, response, body_deadline).await; let origin_work = finish_segment_origin_attempt(context, guard).await; commit_result.map(|metadata| SegmentFetchCommit { content_length: metadata.size, @@ -742,11 +734,9 @@ async fn fetch_segment_with_retries_into_cache( async fn commit_segment_response_into_cache( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, - response: reqwest::Response, + response: DecodedHttpResponse, + body_deadline: HlsOriginResourceBodyDeadline, ) -> Result { - let deadline = hls_object_body_deadline(policy.origin_segment_timeout_ms); - let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); let proxy_session_id = context.session.read().await.proxy_session_id.clone(); let repair_context = HlsSegmentRepairObjectContext { source: HlsSegmentRepairSource::Normal, @@ -765,15 +755,15 @@ async fn commit_segment_response_into_cache( }; context .segment_repair - .commit_origin_response(&context.segment_cache, &snapshot.cache_key, stream_reader, deadline, repair_context) + .commit_origin_response( + &context.segment_cache, + &snapshot.cache_key, + response.body, + body_deadline.deadline(), + repair_context, + ) .await - .map_err(|err| { - if err.kind() == io::ErrorKind::TimedOut { - SegmentFetchError::Timeout - } else { - SegmentFetchError::cache_commit(&err) - } - }) + .map_err(|err| SegmentFetchError::cache_body(&err)) } fn build_segment_origin_headers( @@ -798,9 +788,13 @@ mod tests { model::HlsSegmentRepairConfig, processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, }; + use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZstdEncoder}; use axum::http::{header, HeaderMap, HeaderValue}; + use shared::model::HlsSegmentRepairMode; use std::{ collections::VecDeque, + io::Cursor, + path::{Path, PathBuf}, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -808,11 +802,10 @@ mod tests { time::Duration, }; use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, + io::{AsyncReadExt, AsyncWriteExt, BufReader}, net::TcpListener, sync::Mutex, }; - use shared::model::HlsSegmentRepairMode; const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; @@ -913,10 +906,8 @@ mod tests { origin_io: None, }; - let snapshot = worker - .next_fetch_snapshot(&context, 11, &SegmentFetchPolicy::default()) - .await - .expect("segment snapshot"); + let snapshot = + worker.next_fetch_snapshot(&context, 11, &SegmentFetchPolicy::default()).await.expect("segment snapshot"); assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/media/seg001.ts"); assert_eq!(snapshot.fetch_ref.byte_range, None); @@ -934,6 +925,128 @@ mod tests { fn drop(&mut self) { self.task.abort(); } } + #[derive(Clone)] + struct TestOriginResponse { + status: u16, + headers: Vec<(String, String)>, + body: Vec, + } + + impl TestOriginResponse { + fn encoded(content_encoding: &str, body: Vec) -> Self { + Self { status: 200, headers: vec![("Content-Encoding".to_string(), content_encoding.to_string())], body } + } + } + + #[derive(Debug, Clone, Copy)] + enum TestContentEncoding { + Gzip, + RawDeflate, + Brotli, + Zstd, + } + + impl TestContentEncoding { + const fn header_value(self) -> &'static str { + match self { + Self::Gzip => "gzip", + Self::RawDeflate => "deflate", + Self::Brotli => "br", + Self::Zstd => "zstd", + } + } + } + + async fn encode_test_body(body: &[u8], encoding: TestContentEncoding) -> Vec { + let reader = BufReader::new(Cursor::new(body.to_vec())); + let mut encoded = Vec::new(); + match encoding { + TestContentEncoding::Gzip => GzipEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::RawDeflate => DeflateEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::Brotli => BrotliEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::Zstd => ZstdEncoder::new(reader).read_to_end(&mut encoded).await, + } + .expect("test body encodes"); + encoded + } + + async fn spawn_sequence_response_server(responses: Vec) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let responses = Arc::new(Mutex::new(VecDeque::from(responses))); + let task_active = Arc::clone(&active); + let task_max = Arc::clone(&max_active); + let task_requests = Arc::clone(&requests); + let task_responses = Arc::clone(&responses); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let active = Arc::clone(&task_active); + let max_active = Arc::clone(&task_max); + let requests = Arc::clone(&task_requests); + let responses = Arc::clone(&task_responses); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(current, Ordering::SeqCst); + let response = responses.lock().await.pop_front().unwrap_or(TestOriginResponse { + status: 500, + headers: Vec::new(), + body: Vec::new(), + }); + active.fetch_sub(1, Ordering::SeqCst); + let reason = if response.status == 200 { "OK" } else { "Status" }; + let mut head = + format!("HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n", response.status, response.body.len()); + for (name, value) in response.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&response.body).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), max_active, requests, task } + } + + fn temp_cache_files(root: &Path) -> Vec { + fn visit(path: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + visit(&path, found); + } else if path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.contains(".tmp.")) { + found.push(path); + } + } + } + + let mut found = Vec::new(); + visit(root, &mut found); + found + } + async fn spawn_segment_server(delay_ms: u64) -> TestSegmentServer { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); let addr = listener.local_addr().expect("local addr"); @@ -1189,6 +1302,21 @@ mod tests { } } + async fn committed_segment( + context: &SegmentFetchContext, + proxy_seq: u64, + ) -> (crate::api::model::SegmentCacheKey, Vec) { + let cache_key = context.session.read().await.segments.get(&proxy_seq).expect("segment").cache_key.clone(); + let metadata = context + .segment_cache + .metadata(&cache_key) + .await + .expect("cache metadata reads") + .expect("cache object exists"); + let bytes = tokio::fs::read(metadata.path).await.expect("cache object reads"); + (cache_key, bytes) + } + #[tokio::test] async fn demand_fetch_writes_cache_and_sets_ready_after_commit() { let server = spawn_segment_server(0).await; @@ -1207,6 +1335,133 @@ mod tests { assert!(matches!(session.segments.get(&1).expect("segment").status, SegmentCacheStatus::Ready { .. })); } + #[tokio::test] + async fn segment_declared_codings_decode_to_identity_before_cache_and_preserve_opaque_bytes() { + let identity_bytes = b"\x00\xffopaque-hls-ciphertext\x1f\x8bmedia".to_vec(); + + for encoding in [ + TestContentEncoding::Gzip, + TestContentEncoding::RawDeflate, + TestContentEncoding::Brotli, + TestContentEncoding::Zstd, + ] { + let encoded = encode_test_body(&identity_bytes, encoding).await; + let server = + spawn_sequence_response_server(vec![TestOriginResponse::encoded(encoding.header_value(), encoded)]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready, "failed for {encoding:?}"); + let (cache_key, cached) = committed_segment(&context, 1).await; + assert_eq!(cached, identity_bytes, "cache retained HTTP coding for {encoding:?}"); + let mut range = context.segment_cache.open_range(&cache_key, 5).await.expect("cache range opens"); + let mut ranged = Vec::new(); + range.read_to_end(&mut ranged).await.expect("cache range reads"); + assert_eq!(ranged, identity_bytes[5..], "range did not address Identity bytes for {encoding:?}"); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1, "unexpected retries for {encoding:?}"); + assert!( + requests[0].to_ascii_lowercase().contains("accept-encoding: identity"), + "request did not enforce identity for {encoding:?}" + ); + } + } + + #[tokio::test] + async fn segment_decoder_failure_retries_then_commits_and_cleans_temp_file() { + let identity_bytes = b"identity-media-after-retry".to_vec(); + let valid = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; + let mut corrupt = valid.clone(); + corrupt.truncate(corrupt.len() / 2); + let server = spawn_sequence_response_server(vec![ + TestOriginResponse::encoded("gzip", corrupt), + TestOriginResponse::encoded("gzip", valid), + ]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + assert_eq!(committed_segment(&context, 1).await.1, identity_bytes); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); + drop(requests); + assert!(!context.segment_cache.has_active_temp_files().await); + assert!(temp_cache_files(temp_dir.path()).is_empty()); + } + + #[tokio::test] + async fn segment_decoder_failure_exhausts_attempt_budget_without_cache_or_temp_file() { + let valid = encode_test_body(b"never committed", TestContentEncoding::Gzip).await; + let mut corrupt = valid; + corrupt.truncate(corrupt.len() / 2); + let server = spawn_sequence_response_server( + (0..5).map(|_| TestOriginResponse::encoded("gzip", corrupt.clone())).collect(), + ) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::TimedOut); + assert_eq!(server.requests.lock().await.len(), 5); + let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); + assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); + assert!(!context.segment_cache.has_active_temp_files().await); + assert!(temp_cache_files(temp_dir.path()).is_empty()); + } + + #[tokio::test] + async fn segment_decoded_object_limit_is_authoritative_and_non_retryable() { + let identity_bytes = vec![b'x'; 512]; + let encoded = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; + assert!(encoded.len() < 64, "fixture must be smaller on the wire than the decoded limit"); + let server = spawn_sequence_response_server(vec![TestOriginResponse::encoded("gzip", encoded)]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + context.segment_cache.update_cache_limits(64, 64); + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); + assert_eq!(server.requests.lock().await.len(), 1); + let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); + assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); + assert!(!context.segment_cache.has_active_temp_files().await); + assert!(temp_cache_files(temp_dir.path()).is_empty()); + } + #[tokio::test] async fn demand_fetch_is_blocked_for_gc_marked_session() { let server = spawn_segment_server(0).await; diff --git a/backend/src/api/model/hls_cache/segment_repair.rs b/backend/src/api/model/hls_cache/segment_repair.rs index fc30d8a7c..eb1eccd65 100644 --- a/backend/src/api/model/hls_cache/segment_repair.rs +++ b/backend/src/api/model/hls_cache/segment_repair.rs @@ -5,11 +5,11 @@ use super::{ CachedSegmentMetadata, HlsAccessLeaseId, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, StagedCacheObject, }; use crate::model::{HlsCorruptSegmentWatchdogConfig, HlsSegmentRepairConfig}; -use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; use arc_swap::ArcSwap; use log::{debug, warn}; use serde_json::Value; use sha2::{Digest, Sha256}; +use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; use std::{ collections::{HashMap, HashSet, VecDeque}, fmt, io, @@ -542,14 +542,14 @@ impl HlsSegmentRepairManager { segment_cache: &HlsSegmentCache, key: &K, reader: R, - deadline: Duration, + deadline: tokio::time::Instant, context: HlsSegmentRepairObjectContext, ) -> io::Result where K: HlsCacheObjectKey, R: AsyncRead + Unpin, { - let raw = segment_cache.stage_temp_with_timeout(key, reader, deadline).await?; + let raw = segment_cache.stage_temp_with_deadline(key, reader, deadline).await?; let runtime = self.runtime.load_full(); if !runtime.postprocessing_enabled() { return segment_cache.commit_staged(key, raw).await; @@ -940,7 +940,8 @@ impl HlsSegmentRepairManager { return Ok(None); } }; - let validation = validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); + let validation = + validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); if let Err(reason) = validation { debug_repair_event(context, executed_level, "validation failed", Some(&reason)); let _ = fs::remove_file(&fixed_path).await; @@ -1176,10 +1177,7 @@ struct RepairRemuxStreamSelection { #[cfg(test)] impl RepairRemuxStreamSelection { fn preserve_all(probe: &SegmentProbe) -> Self { - Self { - mapped_streams: probe.streams.iter().map(|stream| stream.index).collect(), - dropped_streams: Vec::new(), - } + Self { mapped_streams: probe.streams.iter().map(|stream| stream.index).collect(), dropped_streams: Vec::new() } } } @@ -1774,10 +1772,8 @@ fn parse_probe(json: &str, warnings: WarningCounters) -> Result Result { mapped_streams.push(stream.index); } - SegmentProbeStreamType::Video => dropped_streams.push(RepairRemuxDroppedStream { - index: stream.index, - reason: "invalid-video-parameters", - }), - SegmentProbeStreamType::Audio => dropped_streams.push(RepairRemuxDroppedStream { - index: stream.index, - reason: "invalid-audio-parameters", - }), - SegmentProbeStreamType::Other => dropped_streams.push(RepairRemuxDroppedStream { - index: stream.index, - reason: "unsupported-stream-type", - }), + SegmentProbeStreamType::Video => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-video-parameters" }), + SegmentProbeStreamType::Audio => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-audio-parameters" }), + SegmentProbeStreamType::Other => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "unsupported-stream-type" }), } } if !has_video { @@ -2001,10 +1991,10 @@ impl fmt::Display for RepairStatus { #[cfg(test)] mod tests { use super::{ - detect_video_codec, parse_ffmpeg_warnings, parse_probe, repair_object_metadata_key, select_repair_remux_streams, - sha256_file, validate_repair, HlsRepairObjectMetadata, HlsRepairRenderedObjectId, HlsSegmentRepairManager, - HlsSegmentRepairObjectContext, HlsSegmentRepairSource, RepairIdentity, RepairRemuxStreamSelection, - RepairStatus, RepairVideoCodec, WarningCounters, REPAIR_METADATA_MAX_ENTRIES, + detect_video_codec, parse_ffmpeg_warnings, parse_probe, repair_object_metadata_key, + select_repair_remux_streams, sha256_file, validate_repair, HlsRepairObjectMetadata, HlsRepairRenderedObjectId, + HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, RepairIdentity, + RepairRemuxStreamSelection, RepairStatus, RepairVideoCodec, WarningCounters, REPAIR_METADATA_MAX_ENTRIES, }; use crate::{ api::model::{ @@ -2013,8 +2003,8 @@ mod tests { }, model::HlsSegmentRepairConfig, }; - use std::sync::Arc; use shared::model::HlsSegmentRepairMode; + use std::sync::Arc; fn repair_config(mode: HlsSegmentRepairMode, apply_to_first_segments: u8) -> HlsSegmentRepairConfig { HlsSegmentRepairConfig { @@ -2550,10 +2540,7 @@ mod tests { assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); assert_eq!(selected_repair_mode(&manager, &same_rendered_object_other_fetch_uri).await, None); - assert_eq!( - selected_repair_mode(&manager, &second_rendered_object).await, - Some(HlsSegmentRepairMode::Low) - ); + assert_eq!(selected_repair_mode(&manager, &second_rendered_object).await, Some(HlsSegmentRepairMode::Low)); } #[tokio::test] diff --git a/backend/src/api/model/hls_cache/segment_watchdog.rs b/backend/src/api/model/hls_cache/segment_watchdog.rs index 32216cc5f..c756efabe 100644 --- a/backend/src/api/model/hls_cache/segment_watchdog.rs +++ b/backend/src/api/model/hls_cache/segment_watchdog.rs @@ -8,9 +8,9 @@ use super::{ CachedSegmentMetadata, HlsCacheObjectKey, HlsSegmentCache, StagedCacheObject, }; use crate::model::HlsCorruptSegmentWatchdogConfig; -use shared::model::HlsCorruptSegmentWatchdogMode; use log::debug; use serde_json::Value; +use shared::model::HlsCorruptSegmentWatchdogMode; use std::{ collections::{HashMap, VecDeque}, ffi::OsString, @@ -261,7 +261,12 @@ impl HlsCorruptSegmentWatchdogManager { Some(format!("unsupported_container:{extension}")), ) .await; - debug_watchdog_event(context, config.mode, "detected corrupt", Some("action=raw_commit reason=unsupported_container")); + debug_watchdog_event( + context, + config.mode, + "detected corrupt", + Some("action=raw_commit reason=unsupported_container"), + ); return segment_cache.commit_staged(key, raw).await; } let fixed_path = watchdog_output_path(&raw.path); diff --git a/backend/src/api/model/hls_cache/session.rs b/backend/src/api/model/hls_cache/session.rs index f97f3a7b5..90b0a63fe 100644 --- a/backend/src/api/model/hls_cache/session.rs +++ b/backend/src/api/model/hls_cache/session.rs @@ -1,10 +1,10 @@ use super::{ build_proxy_session_id, classify_account_binding_protection, HlsAccountBindingProtection, HlsAccountOverlapTiming, - HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, HlsFreshManifestRequiredReason, - HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountRebindState, HlsOriginSource, HlsSessionKey, - HlsBoundAccountAcquireErrorKind, MapCacheStatus, MapEntry, OriginMapKey, OriginRefreshState, OriginSegmentKey, - ProxyMapId, ProxySessionId, RenderPolicy, RenderedManifest, SegmentCacheStatus, SegmentEntry, - SegmentFetchPriority, SegmentPrefetchQueue, TransientPassthroughState, + HlsBoundAccountAcquireErrorKind, HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, + HlsFreshManifestRequiredReason, HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountRebindState, + HlsOriginSource, HlsSessionKey, MapCacheStatus, MapEntry, OriginMapKey, OriginRefreshState, OriginSegmentKey, + ProxyMapId, ProxySessionId, RenderPolicy, RenderedManifest, SegmentCacheStatus, SegmentEntry, SegmentFetchPriority, + SegmentPrefetchQueue, TransientPassthroughState, }; use axum::http::{HeaderMap, StatusCode}; use std::{ diff --git a/backend/src/api/model/hls_cache/timeline.rs b/backend/src/api/model/hls_cache/timeline.rs index e54e5720f..046cac9f1 100644 --- a/backend/src/api/model/hls_cache/timeline.rs +++ b/backend/src/api/model/hls_cache/timeline.rs @@ -271,10 +271,10 @@ impl TimelineDraft { self.target_duration = manifest.target_duration; self.independent_segments = manifest.independent_segments; self.apply_forward_jump(manifest)?; - let epoch_transition_discontinuity = self.apply_origin_epoch_transition_for_manifest(manifest, origin_epoch_handoff)?; + let epoch_transition_discontinuity = + self.apply_origin_epoch_transition_for_manifest(manifest, origin_epoch_handoff)?; - let mut mark_handoff_discontinuity = - handoff_discontinuity_sequence.is_some() || epoch_transition_discontinuity; + let mut mark_handoff_discontinuity = handoff_discontinuity_sequence.is_some() || epoch_transition_discontinuity; let mut manifest_head_proxy_seq = None; let mut manifest_tail_proxy_seq = None; for parsed in &manifest.segments { @@ -295,19 +295,15 @@ impl TimelineDraft { origin_epoch_handoff: bool, ) -> Result { if origin_epoch_handoff && self.should_start_new_origin_epoch_for_handoff(manifest) { - let next_origin_seq = manifest - .segments - .first() - .map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); + let next_origin_seq = + manifest.segments.first().map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); self.start_new_origin_epoch(next_origin_seq, OriginEpochTransitionReason::Handoff)?; return Ok(true); } if self.should_start_new_origin_epoch_for_rollover(manifest) { - let next_origin_seq = manifest - .segments - .first() - .map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); + let next_origin_seq = + manifest.segments.first().map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); self.start_new_origin_epoch(next_origin_seq, OriginEpochTransitionReason::Rollover)?; return Ok(true); } @@ -484,9 +480,7 @@ fn proxy_extension_from_url(url: &str, allowed_extensions: &[&str]) -> Option HlsTransientObjectFetchFailure { - match error { - HlsOriginResourceFetchError::PermanentStatus(status) - | HlsOriginResourceFetchError::NonRetryableStatus(status) => { - HlsTransientObjectFetchFailure::Permanent { status: Some(*status) } - } - HlsOriginResourceFetchError::InvalidOriginUrl - | HlsOriginResourceFetchError::InvalidByteRange - | HlsOriginResourceFetchError::UnexpectedByteRangeStatus => { - HlsTransientObjectFetchFailure::Permanent { status: None } - } - HlsOriginResourceFetchError::RetryableStatus(_) - | HlsOriginResourceFetchError::Transport(_) - | HlsOriginResourceFetchError::Redirect - | HlsOriginResourceFetchError::Timeout - | HlsOriginResourceFetchError::CacheCommit(_) => HlsTransientObjectFetchFailure::Retryable, - HlsOriginResourceFetchError::ProviderUnavailable(kind) if kind.is_retryable_resource_failure() => { - HlsTransientObjectFetchFailure::Retryable - } - HlsOriginResourceFetchError::ProviderUnavailable(_) => HlsTransientObjectFetchFailure::Permanent { status: None }, + if let Some(status) = error.permanent_status() { + return HlsTransientObjectFetchFailure::Permanent { status: Some(status) }; } + if !error.retryable_failure() { + return HlsTransientObjectFetchFailure::Permanent { status: None }; + } + HlsTransientObjectFetchFailure::Retryable } pub fn hls_transient_resource_fetch_kind(resource_kind: TransientResourceKind) -> HlsResourceFetchKind { @@ -51,6 +46,7 @@ pub fn hls_transient_resource_fetch_kind(resource_kind: TransientResourceKind) - TransientResourceKind::Key => HlsResourceFetchKind::Key, TransientResourceKind::Map => HlsResourceFetchKind::Map, TransientResourceKind::Segment => HlsResourceFetchKind::Segment, + TransientResourceKind::Part => HlsResourceFetchKind::Part, TransientResourceKind::Other => HlsResourceFetchKind::Other, } } @@ -59,11 +55,17 @@ fn build_hls_transient_resource_fetch_target( resolved_origin_uri: &str, origin_headers: &HeaderMap, origin_provider_session_headers: &HeaderMap, - range_header: Option, + mode: HlsTransientOriginFetchMode, resource_id: &str, resource_kind: TransientResourceKind, -) -> HlsOriginResourceFetchTarget { - HlsOriginResourceFetchTarget { +) -> Result { + let (range_header, byte_range_expectation) = match mode { + HlsTransientOriginFetchMode::CacheFullObject => (None, HlsOriginByteRangeExpectation::FullObject), + HlsTransientOriginFetchMode::DirectPassthrough { client_range } => { + (client_range, HlsOriginByteRangeExpectation::AnySuccess) + } + }; + Ok(HlsOriginResourceFetchTarget { kind: hls_transient_resource_fetch_kind(resource_kind), source: HlsResourceFetchSource::Transient, object_id: resource_id.to_string(), @@ -72,9 +74,15 @@ fn build_hls_transient_resource_fetch_target( origin_headers, origin_provider_session_headers, range_header, - ), - byte_range_expectation: HlsOriginByteRangeExpectation::AnySuccess, - } + )?, + byte_range_expectation, + }) +} + +/// Keeps full-object cache fills separate from decoded client-range passthrough. +enum HlsTransientOriginFetchMode { + CacheFullObject, + DirectPassthrough { client_range: Option }, } pub struct HlsTransientOriginFetchRequest { @@ -89,10 +97,18 @@ pub struct HlsTransientOriginFetchRequest { pub session_log_id: String, } +/// A decoded direct-origin response together with the attempt and guards that own its origin work. +pub struct HlsTransientDecodedOriginResponse { + pub decoded: DecodedHttpResponse, + pub body_deadline: HlsOriginResourceBodyDeadline, + pub attempt: HlsResourceFetchAttempt, + pub guard: G, +} + pub async fn fetch_hls_transient_origin_response_with_attempt_prepare( request: HlsTransientOriginFetchRequest, prepare_attempt: P, -) -> Result<(reqwest::Response, G), HlsOriginResourceFetchError> +) -> Result, HlsOriginResourceFetchError> where G: Send + 'static, P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, @@ -101,10 +117,10 @@ where &request.resolved_origin_uri, &request.origin_headers, &request.origin_provider_session_headers, - request.range_header, + HlsTransientOriginFetchMode::DirectPassthrough { client_range: request.range_header }, request.resource_file.resource_id.0.as_str(), request.resource_kind, - ); + )?; run_hls_origin_resource_retry_loop_with_attempt_prepare( target, request.clients, @@ -112,7 +128,9 @@ where &request.session_log_id, prepare_attempt, |guard| async move { drop(guard) }.boxed(), - |response, _attempt, guard| async move { Ok((response, guard)) }.boxed(), + |decoded, attempt, body_deadline, guard| { + async move { Ok(HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard }) }.boxed() + }, ) .await } @@ -182,8 +200,11 @@ fn transient_object_cache_action( if matches!(resource.kind, TransientResourceKind::Key) { return HlsTransientObjectCacheAction::PassthroughNoCache; } - let cache_key = - TransientPassthroughState::transient_object_key(proxy_session_id, &resource.id, resource_file.extension.clone()); + let cache_key = TransientPassthroughState::transient_object_key( + proxy_session_id, + &resource.id, + resource_file.extension.clone(), + ); if session.transient.ready_object(&cache_key, now_ms).is_some() { return HlsTransientObjectCacheAction::ServeReady; } @@ -255,9 +276,7 @@ pub struct HlsTransientCacheCommitContext { pub resource: TransientResourceRef, pub resource_file: TransientResourceFile, pub cache_key: TransientObjectCacheKey, - pub range_header: Option, pub cache_duration_ms: u64, - pub origin_segment_timeout_ms: u64, } pub struct HlsTransientOriginCacheFetchRequest { @@ -277,10 +296,10 @@ where &request.fetch.resolved_origin_uri, &request.fetch.origin_headers, &request.fetch.origin_provider_session_headers, - request.fetch.range_header, + HlsTransientOriginFetchMode::CacheFullObject, request.fetch.resource_file.resource_id.0.as_str(), request.fetch.resource_kind, - ); + )?; let commit = request.commit; run_hls_origin_resource_retry_loop_with_attempt_prepare( target, @@ -289,10 +308,10 @@ where &request.fetch.session_log_id, prepare_attempt, |guard| async move { drop(guard) }.boxed(), - move |response, _attempt, guard| { + move |decoded, _attempt, body_deadline, guard| { let commit = commit.clone(); async move { - let result = commit_hls_transient_origin_response_attempt(commit, response).await; + let result = commit_hls_transient_origin_response_attempt(commit, decoded, body_deadline).await; drop(guard); result } @@ -304,17 +323,16 @@ where async fn commit_hls_transient_origin_response_attempt( context: HlsTransientCacheCommitContext, - response: reqwest::Response, + decoded: DecodedHttpResponse, + body_deadline: HlsOriginResourceBodyDeadline, ) -> Result<(), HlsOriginResourceFetchError> { - let response_headers = response.headers().clone(); - let content_type = response_headers + let content_type = decoded + .headers .get(header::CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .map(str::to_string) .or_else(|| context.resource.content_type_hint.clone()) .unwrap_or_else(|| "application/octet-stream".to_string()); - let deadline = hls_object_body_deadline(context.origin_segment_timeout_ms); - let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); let repair_context = HlsSegmentRepairObjectContext { source: HlsSegmentRepairSource::Transient, proxy_session_id: context.session.read().await.proxy_session_id.clone(), @@ -327,27 +345,24 @@ async fn commit_hls_transient_origin_response_attempt( origin_fetch_uri_for_diagnostics: context.resource.resolved_origin_uri.clone(), media_sequence: None, discontinuity_sequence: None, - complete_object: is_hls_transient_full_object_cacheable_request(context.range_header.as_ref()), + complete_object: true, encrypted: context.resource.kind == TransientResourceKind::Key, custom_response: false, }; let commit = Box::pin(context.segment_repair.commit_origin_response( &context.segment_cache, &context.cache_key, - stream_reader, - deadline, + decoded.body, + body_deadline.deadline(), repair_context, )) .await; let ready_at_ms = current_time_millis(); let metadata = match commit { Ok(metadata) => metadata, - Err(err) if err.kind() == io::ErrorKind::TimedOut => return Err(HlsOriginResourceFetchError::Timeout), - Err(err) => return Err(HlsOriginResourceFetchError::cache_commit(&err)), + Err(err) => return Err(HlsOriginResourceFetchError::cache_body(&err)), }; - let expires_at_ms = ready_at_ms - .saturating_add(context.cache_duration_ms) - .max(context.resource.expires_at_ms); + let expires_at_ms = ready_at_ms.saturating_add(context.cache_duration_ms).max(context.resource.expires_at_ms); context.session.write().await.transient.mark_object_ready( &context.cache_key, content_type, @@ -358,20 +373,110 @@ async fn commit_hls_transient_origin_response_attempt( Ok(()) } -#[allow(clippy::too_many_arguments)] +/// Context retained until a direct transient response body reaches one terminal outcome. +pub struct HlsTransientDirectResponseContext { + pub hls_proxy: Arc, + pub session: HlsSessionHandle, + pub resource: TransientResourceRef, + pub policy: SegmentFetchPolicy, + pub media_activity_marker: Option, + pub now_ms: u64, + pub proxy_session_id: ProxySessionId, +} + +#[derive(Clone, Copy)] +enum HlsTransientDirectStreamOutcome { + CleanEof, + OriginBodyFailure, + ClientAborted, +} + +// Sole owner of the direct-body outcome: EOF is success, origin read failures +// count as segment failures, and downstream cancellation remains neutral. +struct HlsTransientDirectResponseFinalizer { + context: Option, +} + +struct HlsTransientDirectResponseLifecycleContext { + hls_proxy: Arc, + session: HlsSessionHandle, + resource: TransientResourceRef, + policy: SegmentFetchPolicy, + proxy_session_id: ProxySessionId, +} + +impl HlsTransientDirectResponseFinalizer { + fn new(context: HlsTransientDirectResponseLifecycleContext) -> Self { + let context = (context.resource.kind == TransientResourceKind::Segment).then_some(context); + Self { context } + } + + async fn finish(&mut self, outcome: HlsTransientDirectStreamOutcome) { + let Some(completion) = self.begin_finish(outcome) else { + return; + }; + if let Err(error) = completion.await { + warn!( + "HLS transient direct lifecycle task failed: cancelled={} panic={}", + error.is_cancelled(), + error.is_panic() + ); + } + } + + fn begin_finish(&mut self, outcome: HlsTransientDirectStreamOutcome) -> Option> { + let context = self.context.take()?; + if matches!(outcome, HlsTransientDirectStreamOutcome::ClientAborted) { + return None; + } + // Once EOF or an origin-body failure has been observed, its state transition + // must survive cancellation of the downstream body poll. + Some(tokio::spawn(async move { + match outcome { + HlsTransientDirectStreamOutcome::CleanEof => { + record_successful_transient_segment_fetch(&context.session, &context.resource).await; + } + HlsTransientDirectStreamOutcome::OriginBodyFailure => { + let failed_at_ms = current_time_millis(); + if let Some(reason) = record_temporary_transient_segment_fetch_failure( + &context.session, + &context.resource, + &context.policy, + failed_at_ms, + ) + .await + { + context + .hls_proxy + .mark_access_leases_channel_unavailable_for_session( + &context.proxy_session_id, + failed_at_ms, + reason, + ) + .await; + } + } + HlsTransientDirectStreamOutcome::ClientAborted => {} + } + })) + } + + fn finish_client_aborted(&mut self) { self.context.take(); } +} + +impl Drop for HlsTransientDirectResponseFinalizer { + fn drop(&mut self) { + // A body dropped before EOF is a downstream abort. It must release the + // retained guards without changing provider/segment failure state. + self.finish_client_aborted(); + } +} + pub fn hls_transient_origin_response( - response: reqwest::Response, - access: Arc, - origin_io_guard: Option, - media_activity_marker: Option, - now_ms: u64, - proxy_session_id: String, - resource_id: String, - resource_kind: TransientResourceKind, - origin_url: String, - origin_segment_timeout_ms: u64, + response: HlsTransientDecodedOriginResponse>, + context: HlsTransientDirectResponseContext, ) -> axum::response::Response { - let mut builder = axum::response::Response::builder().status(response.status()); + let mut builder = axum::response::Response::builder().status(response.decoded.status); for header_name in [ header::CONTENT_TYPE, header::CONTENT_LENGTH, @@ -381,83 +486,187 @@ pub fn hls_transient_origin_response( header::ETAG, header::LAST_MODIFIED, ] { - if let Some(value) = response.headers().get(&header_name) { + if let Some(value) = response.decoded.headers.get(&header_name) { builder = builder.header(header_name, value.clone()); } } - let guard = HlsTransientReadGuard::new(access, now_ms); + let mut response = try_unwrap_body!(builder.body(hls_transient_direct_body(response, context))); + mark_response_as_uncompressed(&mut response); + response +} + +fn hls_transient_direct_body( + response: HlsTransientDecodedOriginResponse>, + context: HlsTransientDirectResponseContext, +) -> Body { + let HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard: origin_io_guard } = response; + let HlsTransientDirectResponseContext { + hls_proxy, + session, + resource, + policy, + media_activity_marker, + now_ms, + proxy_session_id, + } = context; + + let guard = HlsTransientReadGuard::new(Arc::clone(&resource.access), now_ms); let media_activity_guard = HlsTransientMediaActivityGuard::new(media_activity_marker, now_ms); - let deadline = hls_object_body_deadline(origin_segment_timeout_ms); + let finalizer = HlsTransientDirectResponseFinalizer::new(HlsTransientDirectResponseLifecycleContext { + hls_proxy, + session, + resource: resource.clone(), + policy, + proxy_session_id: proxy_session_id.clone(), + }); + let resource_id = resource.id.0.clone(); + let resource_kind = resource.kind; let stream = futures::stream::unfold( ( - response.bytes_stream(), + ReaderStream::new(decoded.body), Some(guard), origin_io_guard, Some(media_activity_guard), + finalizer, Box::pin(sleep(hls_client_body_send_deadline())), false, ), - move |(mut stream, guard, origin_io_guard, media_activity_guard, mut send_deadline, finished)| { - let proxy_session_id = proxy_session_id.clone(); + move |( + mut stream, + guard, + origin_io_guard, + media_activity_guard, + mut finalizer, + mut send_deadline, + finished, + )| { + let proxy_session_id = proxy_session_id.0.clone(); let resource_id = resource_id.clone(); - let origin_url = origin_url.clone(); async move { if finished { return None; } let next_chunk = tokio::select! { () = send_deadline.as_mut() => { - log_hls_resource_timeout( - &proxy_session_id, - HlsResourceFetchLogContext { - kind: hls_transient_resource_fetch_kind(resource_kind), - source: HlsResourceFetchSource::Transient, - object_id: &resource_id, - origin_url: Some(&origin_url), - }, - HlsResourceFetchAttempt { attempt_index: 0, attempts: 1 }, - hls_client_body_send_deadline().as_millis(), - ); + finalizer.finish(HlsTransientDirectStreamOutcome::ClientAborted).await; return Some(( Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")), - (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), + (stream, None, None, None, finalizer, send_deadline, true), )); } - next_chunk = tokio::time::timeout(deadline, stream.next()) => next_chunk, + // Decoder setup is bounded by the absolute attempt deadline before this + // response is built. Once handed to the client, retain the existing + // per-chunk origin-body idle timeout instead of imposing a total-body limit. + next_chunk = tokio::time::timeout(body_deadline.timeout(), stream.next()) => next_chunk, }; match next_chunk { Ok(Some(Ok(chunk))) => { refresh_hls_client_body_send_deadline(send_deadline.as_mut()); - Some((Ok(chunk), (stream, guard, origin_io_guard, media_activity_guard, send_deadline, false))) - } - Ok(Some(Err(err))) => Some(( - Err(io::Error::other(err)), - (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), - )), - Ok(None) => None, - Err(_) => { - log_hls_resource_timeout( - &proxy_session_id, - HlsResourceFetchLogContext { - kind: hls_transient_resource_fetch_kind(resource_kind), - source: HlsResourceFetchSource::Transient, - object_id: &resource_id, - origin_url: Some(&origin_url), - }, - HlsResourceFetchAttempt { attempt_index: 0, attempts: 1 }, - deadline.as_millis(), - ); Some(( - Err(io::Error::new(io::ErrorKind::TimedOut, "transient passthrough body timed out")), - (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), + Ok(chunk), + (stream, guard, origin_io_guard, media_activity_guard, finalizer, send_deadline, false), )) } + Ok(Some(Err(err))) => { + log_hls_resource_body_failure( + &proxy_session_id, + hls_transient_direct_log_context(&resource_id, resource_kind), + attempt, + &err, + body_deadline.timeout().as_millis(), + ); + finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; + Some((Err(err), (stream, None, None, None, finalizer, send_deadline, true))) + } + Ok(None) => { + finalizer.finish(HlsTransientDirectStreamOutcome::CleanEof).await; + None + } + Err(_) => { + let error = io::Error::new(io::ErrorKind::TimedOut, "transient passthrough body timed out"); + log_hls_resource_body_failure( + &proxy_session_id, + hls_transient_direct_log_context(&resource_id, resource_kind), + attempt, + &error, + body_deadline.timeout().as_millis(), + ); + finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; + Some((Err(error), (stream, None, None, None, finalizer, send_deadline, true))) + } } } }, ); - try_unwrap_body!(builder.body(Body::from_stream(stream))) + Body::from_stream(stream) +} + +fn hls_transient_direct_log_context( + resource_id: &str, + resource_kind: TransientResourceKind, +) -> HlsResourceFetchLogContext<'_> { + HlsResourceFetchLogContext { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: resource_id, + origin_url: None, + } +} + +pub async fn record_successful_transient_segment_fetch(session: &HlsSessionHandle, resource: &TransientResourceRef) { + if resource.kind != TransientResourceKind::Segment { + return; + } + let mut session = session.write().await; + if let Some(reset_failures) = session.record_successful_segment_fetch() { + debug!( + "HLS segment temporary failure counter reset: session={} previous_failures={reset_failures}", + safe_proxy_session_id(&session.proxy_session_id) + ); + } +} + +pub async fn record_temporary_transient_segment_fetch_failure( + session: &HlsSessionHandle, + resource: &TransientResourceRef, + policy: &SegmentFetchPolicy, + now_ms: u64, +) -> Option { + if resource.kind != TransientResourceKind::Segment { + return None; + } + let mut session = session.write().await; + let threshold = session.segment_temporary_failure_threshold(policy.permanent_failure_segment_threshold); + match session.record_temporary_segment_fetch_failure( + now_ms, + HlsSegmentFailureObject::Transient { resource_id: resource.id.0.clone() }, + threshold, + ) { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { + debug!( + "HLS segment temporary failure counted: session={} object={} failures={} threshold={}", + safe_proxy_session_id(&session.proxy_session_id), + resource.id.0, + failures, + threshold + ); + None + } + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { + warn!( + "HLS segment temporary failure threshold reached: session={} failures={} threshold={}", + safe_proxy_session_id(&session.proxy_session_id), + failures, + threshold + ); + session.invalidate_queued_origin_work(); + Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectTemporaryFailureThreshold { + failures, + threshold, + }) + } + } } struct HlsTransientReadGuard { @@ -480,9 +689,7 @@ struct HlsTransientMediaActivityGuard { } impl HlsTransientMediaActivityGuard { - fn new(marker: Option, _now_ms: u64) -> Self { - Self { marker } - } + fn new(marker: Option, _now_ms: u64) -> Self { Self { marker } } } impl Drop for HlsTransientMediaActivityGuard { @@ -525,7 +732,9 @@ impl Drop for HlsTransientOriginIoGuard { session.finish_origin_work(started_generation) }; let refresh_reservation = if generation_valid { - session.read().await.should_refresh_origin_reservation(chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default()) + session.read().await.should_refresh_origin_reservation( + chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(), + ) } else { false }; @@ -543,3 +752,994 @@ impl Drop for HlsTransientOriginIoGuard { } fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{ + super::resource_fetch::take_body_failure_log_attempts, + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, + hls_transient_origin_response, record_temporary_transient_segment_fetch_failure, + HlsTransientCacheCommitContext, HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, + HlsTransientDirectResponseFinalizer, HlsTransientDirectResponseLifecycleContext, + HlsTransientDirectStreamOutcome, HlsTransientObjectFetchFailure, HlsTransientOriginCacheFetchRequest, + HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, + }; + use crate::{ + api::{ + api_utils::should_compress_response, + model::{ + HlsAccessLeaseId, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsProxyManager, + HlsSegmentCache, HlsSegmentFailureObject, HlsSegmentRepairManager, HlsSession, HlsSessionHandle, + HlsSessionKey, HlsSessionStore, ProxySessionId, SegmentFetchPolicy, TransientObjectCacheKey, + TransientObjectCacheStatus, TransientObjectFetchDecision, TransientResourceFile, TransientResourceKind, + TransientResourceRef, + }, + }, + model::HlsSegmentRepairConfig, + }; + use async_compression::tokio::write::{BrotliEncoder, GzipEncoder, ZstdEncoder}; + use axum::{ + body::to_bytes, + http::{header, HeaderMap, HeaderValue, StatusCode}, + }; + use futures::FutureExt; + use shared::model::HlsSegmentRepairMode; + use std::{ + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{Mutex, RwLock}, + }; + + struct TestOrigin { + base_url: String, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for TestOrigin { + fn drop(&mut self) { self.task.abort(); } + } + + #[test] + fn storage_full_transient_cache_failure_is_permanent() { + let error = HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28)); + + assert!(matches!( + hls_transient_object_fetch_failure(&error), + HlsTransientObjectFetchFailure::Permanent { status: None } + )); + } + + async fn spawn_test_origin( + status_line: &'static str, + response_headers: Vec<(&'static str, &'static str)>, + body: Vec, + ) -> TestOrigin { + spawn_test_origin_in_chunks(status_line, response_headers, vec![body], Duration::ZERO).await + } + + async fn spawn_test_origin_in_chunks( + status_line: &'static str, + response_headers: Vec<(&'static str, &'static str)>, + body_chunks: Vec>, + inter_chunk_delay: Duration, + ) -> TestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("test origin address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let body_chunks = Arc::new(body_chunks); + let content_length = body_chunks.iter().map(Vec::len).sum::(); + let response_headers = Arc::new(response_headers); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let body_chunks = Arc::clone(&body_chunks); + let response_headers = Arc::clone(&response_headers); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let mut response = format!("HTTP/1.1 {status_line}\r\nContent-Length: {content_length}\r\n"); + for (name, value) in response_headers.iter() { + response.push_str(name); + response.push_str(": "); + response.push_str(value); + response.push_str("\r\n"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + for (index, chunk) in body_chunks.iter().enumerate() { + if index > 0 && !inter_chunk_delay.is_zero() { + tokio::time::sleep(inter_chunk_delay).await; + } + if socket.write_all(chunk).await.is_err() { + return; + } + } + }); + } + }); + + TestOrigin { base_url: format!("http://{addr}"), requests, task } + } + + async fn spawn_retry_then_body_origin( + response_headers: Vec<(&'static str, &'static str)>, + body: Vec, + ) -> TestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("test origin address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let attempts = Arc::new(AtomicUsize::new(0)); + let task_attempts = Arc::clone(&attempts); + let body = Arc::new(body); + let response_headers = Arc::new(response_headers); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let attempts = Arc::clone(&task_attempts); + let body = Arc::clone(&body); + let response_headers = Arc::clone(&response_headers); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + if attempts.fetch_add(1, Ordering::Relaxed) == 0 { + let _ = socket + .write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await; + return; + } + let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); + for (name, value) in response_headers.iter() { + response.push_str(name); + response.push_str(": "); + response.push_str(value); + response.push_str("\r\n"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.write_all(&body).await; + }); + } + }); + + TestOrigin { base_url: format!("http://{addr}"), requests, task } + } + + async fn spawn_zstd_origin(body: Vec) -> TestOrigin { + spawn_test_origin( + "200 OK", + vec![("Content-Encoding", "zstd"), ("Content-Type", "application/octet-stream")], + body, + ) + .await + } + + async fn gzip_encode(body: &[u8]) -> Vec { + let mut encoder = GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() + } + + async fn brotli_encode(body: &[u8]) -> Vec { + let mut encoder = BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() + } + + async fn zstd_encode(body: &[u8]) -> Vec { + let mut encoder = ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() + } + + async fn fetch_direct_decoded( + origin_url: String, + resource_kind: TransientResourceKind, + range_header: Option, + ) -> Result>, HlsOriginResourceFetchError> { + fetch_direct_decoded_with_timeout(origin_url, resource_kind, range_header, 1_000).await + } + + async fn fetch_direct_decoded_with_timeout( + origin_url: String, + resource_kind: TransientResourceKind, + range_header: Option, + origin_segment_timeout_ms: u64, + ) -> Result>, HlsOriginResourceFetchError> { + let resource = TransientResourceRef::new( + resource_kind, + origin_url, + b"rewrite-secret", + 10, + 60_000, + Some("bin".to_string()), + ); + let request = HlsTransientOriginFetchRequest { + resolved_origin_uri: resource.resolved_origin_uri.clone(), + origin_headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + range_header, + resource_file: TransientResourceFile { resource_id: resource.id, extension: "bin".to_string() }, + resource_kind, + clients: HlsOriginResourceClients { + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + use_manual_redirects: false, + }, + policy: SegmentFetchPolicy { + origin_segment_timeout_ms, + retry_delays_ms: [0; 5], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }, + session_log_id: "direct-transient-content-coding-test".to_string(), + }; + fetch_hls_transient_origin_response_with_attempt_prepare(request, |_| async { Ok(None) }.boxed()).await + } + + fn direct_client_response( + response: HlsTransientDecodedOriginResponse>, + resource_kind: TransientResourceKind, + ) -> axum::response::Response { + TestDirectResponseFixture::new(resource_kind, "http://127.0.0.1/origin").response(response) + } + + struct TestDirectResponseFixture { + hls_proxy: Arc, + session: HlsSessionHandle, + resource: TransientResourceRef, + policy: SegmentFetchPolicy, + proxy_session_id: ProxySessionId, + } + + impl TestDirectResponseFixture { + fn new(resource_kind: TransientResourceKind, origin_url: impl Into) -> Self { + let resource = TransientResourceRef::new( + resource_kind, + origin_url, + b"rewrite-secret", + 10, + 60_000, + Some("bin".to_string()), + ); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let session = HlsSession::new(HlsSessionKey::new(1, "direct-test"), b"rewrite-secret", 10); + let proxy_session_id = session.proxy_session_id.clone(); + Self { + hls_proxy: Arc::new(HlsProxyManager::new()), + session: Arc::new(RwLock::new(session)), + resource, + policy, + proxy_session_id, + } + } + + fn response( + &self, + response: HlsTransientDecodedOriginResponse>, + ) -> axum::response::Response { + hls_transient_origin_response(response, self.response_context()) + } + + fn response_context(&self) -> HlsTransientDirectResponseContext { + HlsTransientDirectResponseContext { + hls_proxy: Arc::clone(&self.hls_proxy), + session: Arc::clone(&self.session), + resource: self.resource.clone(), + policy: self.policy.clone(), + media_activity_marker: None, + now_ms: 10, + proxy_session_id: self.proxy_session_id.clone(), + } + } + + fn lifecycle_context(&self) -> HlsTransientDirectResponseLifecycleContext { + HlsTransientDirectResponseLifecycleContext { + hls_proxy: Arc::clone(&self.hls_proxy), + session: Arc::clone(&self.session), + resource: self.resource.clone(), + policy: self.policy.clone(), + proxy_session_id: self.proxy_session_id.clone(), + } + } + + async fn seed_segment_failure(&self) { + assert!(record_temporary_transient_segment_fetch_failure(&self.session, &self.resource, &self.policy, 9) + .await + .is_none()); + } + + async fn segment_failure_count(&self) -> u32 { + self.session.read().await.segment_failure_tracker.consecutive_temporary_failures + } + + async fn wait_for_segment_failure_count(&self, expected: u32) { + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if self.segment_failure_count().await == expected { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("segment failure state reaches expected value"); + } + + async fn last_segment_failure(&self) -> Option { + self.session.read().await.segment_failure_tracker.last_failed_object.clone() + } + } + + struct TestTransientCacheFixture { + _temp_dir: tempfile::TempDir, + segment_cache: Arc, + segment_repair: Arc, + session: HlsSessionHandle, + proxy_session_id: ProxySessionId, + resource: TransientResourceRef, + resource_file: TransientResourceFile, + cache_key: TransientObjectCacheKey, + } + + impl TestTransientCacheFixture { + async fn new(origin_url: String) -> Self { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + origin_url, + b"rewrite-secret", + 10, + 60_000, + Some("ts".to_string()), + ); + let resource_file = TransientResourceFile { resource_id: resource.id.clone(), extension: "ts".to_string() }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"rewrite-secret", 10).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let cache_key = { + let mut session = session.write().await; + match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 10, 60_000) { + TransientObjectFetchDecision::Fetch(cache_key) => cache_key, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new transient resource should start a cache fetch") + } + } + }; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })); + Self { + _temp_dir: temp_dir, + segment_cache, + segment_repair, + session, + proxy_session_id, + resource, + resource_file, + cache_key, + } + } + + fn request(&self, range_header: Option) -> HlsTransientOriginCacheFetchRequest { + HlsTransientOriginCacheFetchRequest { + fetch: HlsTransientOriginFetchRequest { + resolved_origin_uri: self.resource.resolved_origin_uri.clone(), + origin_headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + range_header, + resource_file: self.resource_file.clone(), + resource_kind: self.resource.kind, + clients: HlsOriginResourceClients { + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + use_manual_redirects: false, + }, + policy: SegmentFetchPolicy { + origin_segment_timeout_ms: 1_000, + retry_delays_ms: [0; 5], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }, + session_log_id: self.proxy_session_id.0.clone(), + }, + commit: HlsTransientCacheCommitContext { + segment_cache: Arc::clone(&self.segment_cache), + segment_repair: Arc::clone(&self.segment_repair), + session: Arc::clone(&self.session), + access_lease_id: HlsAccessLeaseId("transient-content-coding-test".to_string()), + resource: self.resource.clone(), + resource_file: self.resource_file.clone(), + cache_key: self.cache_key.clone(), + cache_duration_ms: 60_000, + }, + } + } + } + + struct DropCounter(Arc); + + impl Drop for DropCounter { + fn drop(&mut self) { self.0.fetch_add(1, Ordering::Relaxed); } + } + + #[tokio::test] + async fn direct_key_decodes_declared_gzip_brotli_and_zstd_to_exact_identity_bytes() { + let identity = b"\x00\xffdirect-key-identity\x10\x80"; + let encoded_bodies = [ + ("gzip", gzip_encode(identity).await), + ("br", brotli_encode(identity).await), + ("zstd", zstd_encode(identity).await), + ]; + + for (encoding, encoded) in encoded_bodies { + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], + encoded, + ) + .await; + let decoded = + fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("declared direct key coding decodes"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert!(!should_compress_response(&response)); + assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); + assert!(response.headers().get(header::CONTENT_LENGTH).is_none()); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("decoded key body streams"), + identity.as_slice() + ); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].to_ascii_lowercase().contains("accept-encoding: identity")); + } + } + + #[tokio::test] + async fn direct_map_and_part_or_other_resources_decode_to_identity() { + let identity = b"shared-direct-map-part-other"; + for (resource_kind, encoding, encoded) in [ + (TransientResourceKind::Map, "gzip", gzip_encode(identity).await), + (TransientResourceKind::Part, "br", brotli_encode(identity).await), + (TransientResourceKind::Other, "zstd", zstd_encode(identity).await), + ] { + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], + encoded, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/object.bin", origin.base_url), resource_kind, None) + .await + .expect("declared direct resource coding decodes"); + let response = direct_client_response(decoded, resource_kind); + + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("decoded resource body streams"), + identity.as_slice() + ); + assert_eq!(origin.requests.lock().await.len(), 1); + } + } + + #[tokio::test] + async fn direct_binary_declared_only_leaves_headerless_gzip_magic_unchanged() { + let identity = b"headerless-gzip-representation"; + let headerless_encoded = gzip_encode(identity).await; + let random_magic = vec![0x1f, 0x8b, 0x11, 0x00, 0xff, 0x42, 0x7e]; + + for body in [headerless_encoded, random_magic] { + let origin = + spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], body.clone()).await; + let decoded = + fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("headerless binary is not inspected"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!(to_bytes(response.into_body(), usize::MAX).await.expect("headerless binary streams"), body); + assert_eq!(origin.requests.lock().await.len(), 1); + } + } + + #[tokio::test] + async fn decoded_direct_response_normalizes_headers_and_disables_tower_compression() { + let identity = b"normalized-direct-response"; + let encoded = gzip_encode(identity).await; + let origin = spawn_test_origin( + "200 OK", + vec![ + ("Content-Encoding", "gzip"), + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 0-9/10"), + ("Accept-Ranges", "bytes"), + ("Cache-Control", "private, max-age=5"), + ("ETag", "strong-origin-representation"), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("Set-Cookie", "provider_session=secret"), + ("X-Provider-Secret", "do-not-forward"), + ], + encoded, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("direct response decodes"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + let headers = response.headers(); + + assert!(!should_compress_response(&response)); + for removed in [ + header::CONTENT_ENCODING, + header::CONTENT_LENGTH, + header::CONTENT_RANGE, + header::ACCEPT_RANGES, + header::ETAG, + header::TRANSFER_ENCODING, + header::SET_COOKIE, + ] { + assert!(headers.get(&removed).is_none(), "{removed} must not describe the decoded response"); + } + assert!(headers.get("x-provider-secret").is_none()); + assert_eq!(headers.get(header::CONTENT_TYPE).expect("content type"), "application/octet-stream"); + assert_eq!(headers.get(header::CACHE_CONTROL).expect("cache control"), "private, max-age=5"); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("normalized body streams"), + identity.as_slice() + ); + } + + #[tokio::test] + async fn direct_encoded_partial_content_is_rejected_before_client_response() { + let encoded = zstd_encode(b"encoded-range").await; + let origin = spawn_test_origin( + "206 Partial Content", + vec![ + ("Content-Encoding", "zstd"), + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 2-5/16"), + ], + encoded, + ) + .await; + + let result = fetch_direct_decoded( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .await; + + assert!(matches!(result, Err(HlsOriginResourceFetchError::ContentCoding(_)))); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(request.contains("range: bytes=2-5")); + } + + #[tokio::test] + async fn direct_identity_partial_content_preserves_consistent_range_headers() { + let identity = b"part"; + let origin = spawn_test_origin( + "206 Partial Content", + vec![ + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 2-5/16"), + ("Accept-Ranges", "bytes"), + ], + identity.to_vec(), + ) + .await; + let decoded = fetch_direct_decoded( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .await + .expect("identity partial response is allowed"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers().get(header::CONTENT_LENGTH).expect("content length"), "4"); + assert_eq!(response.headers().get(header::CONTENT_RANGE).expect("content range"), "bytes 2-5/16"); + assert_eq!(response.headers().get(header::ACCEPT_RANGES).expect("accept ranges"), "bytes"); + assert!(!should_compress_response(&response)); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("identity partial body streams"), + identity.as_slice() + ); + assert_eq!(origin.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn direct_fetch_propagates_retry_attempt_and_identity_to_client_response() { + let identity = b"successful-second-attempt"; + let origin = + spawn_retry_then_body_origin(vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let decoded = fetch_direct_decoded(origin_url.clone(), TransientResourceKind::Segment, None) + .await + .expect("second origin attempt succeeds"); + + assert_eq!(decoded.attempt.attempt_index, 1); + assert_eq!(decoded.attempt.attempts, 5); + + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url); + let response = fixture.response(decoded); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("retried response body streams"), + identity.as_slice() + ); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); + } + + #[tokio::test] + async fn direct_segment_success_resets_failure_state_only_after_clean_eof() { + let identity = b"complete-segment"; + let origin = + spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + fixture.seed_segment_failure().await; + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + + assert_eq!(fixture.segment_failure_count().await, 1, "headers alone must not report body success"); + assert_eq!(fixture.resource.access.active_readers(), 1); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("complete segment streams"), + identity.as_slice() + ); + fixture.wait_for_segment_failure_count(0).await; + assert_eq!(fixture.resource.access.active_readers(), 0); + } + + #[tokio::test] + async fn direct_terminal_transition_survives_dropped_completion_waiter() { + let fixture = + TestDirectResponseFixture::new(TransientResourceKind::Segment, "http://127.0.0.1/cancelled-body-poll.ts"); + fixture.seed_segment_failure().await; + let session_lock = fixture.session.write().await; + let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); + + let transition = finalizer + .begin_finish(HlsTransientDirectStreamOutcome::CleanEof) + .expect("clean EOF starts an owned transition"); + drop(finalizer); + drop(transition); + drop(session_lock); + + fixture.wait_for_segment_failure_count(0).await; + } + + #[tokio::test] + async fn direct_segment_decoder_failure_after_retry_uses_selected_attempt_once() { + let _ = take_body_failure_log_attempts(); + let mut truncated = gzip_encode(b"decoder failure after response headers").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_retry_then_body_origin( + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("decoder setup succeeds before streaming"); + assert_eq!(decoded.attempt.attempt_index, 1); + assert_eq!(decoded.attempt.attempts, 5); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + fixture.wait_for_segment_failure_count(1).await; + assert_eq!( + fixture.last_segment_failure().await, + Some(HlsSegmentFailureObject::Transient { resource_id: fixture.resource.id.0.clone() }) + ); + tokio::task::yield_now().await; + assert_eq!(fixture.segment_failure_count().await, 1, "terminal body failure must not finalize twice"); + assert_eq!(fixture.resource.access.active_readers(), 0); + assert_eq!(origin.requests.lock().await.len(), 2, "streaming failure must not trigger a hidden retry"); + assert_eq!( + take_body_failure_log_attempts(), + vec![(1, 5)], + "test hook intentionally records the zero-based raw index of the selected second attempt" + ); + } + + #[tokio::test] + async fn direct_non_segment_finalizers_do_not_start_lifecycle_tasks() { + for kind in [ + TransientResourceKind::Key, + TransientResourceKind::Map, + TransientResourceKind::Part, + TransientResourceKind::Other, + ] { + for outcome in + [HlsTransientDirectStreamOutcome::CleanEof, HlsTransientDirectStreamOutcome::OriginBodyFailure] + { + let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/non-segment.bin"); + let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); + + assert!(finalizer.begin_finish(outcome).is_none(), "{kind:?} must not start a segment lifecycle task"); + } + } + } + + #[tokio::test] + async fn direct_segment_origin_idle_timeout_is_counted_exactly_once() { + let origin = spawn_test_origin_in_chunks( + "200 OK", + vec![("Content-Type", "application/octet-stream")], + vec![b"first".to_vec(), b"late".to_vec()], + Duration::from_millis(200), + ) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + let decoded = fetch_direct_decoded_with_timeout(origin_url, TransientResourceKind::Segment, None, 50) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + fixture.wait_for_segment_failure_count(1).await; + tokio::task::yield_now().await; + assert_eq!(fixture.segment_failure_count().await, 1); + assert_eq!(fixture.resource.access.active_readers(), 0); + } + + #[tokio::test] + async fn dropping_direct_segment_body_is_client_abort_without_state_transition() { + let _ = take_body_failure_log_attempts(); + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Type", "application/octet-stream")], + b"unconsumed-segment".to_vec(), + ) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + fixture.seed_segment_failure().await; + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + assert_eq!(fixture.resource.access.active_readers(), 1); + + drop(response); + + assert_eq!(fixture.segment_failure_count().await, 1); + assert_eq!(fixture.resource.access.active_readers(), 0); + assert_eq!(origin.requests.lock().await.len(), 1); + assert!(take_body_failure_log_attempts().is_empty()); + } + + #[tokio::test] + async fn direct_non_segment_body_failures_are_failure_tracker_neutral() { + for kind in [ + TransientResourceKind::Key, + TransientResourceKind::Map, + TransientResourceKind::Part, + TransientResourceKind::Other, + ] { + let mut truncated = gzip_encode(b"non-segment decoder failure").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let origin_url = format!("{}/object.bin", origin.base_url); + let fixture = TestDirectResponseFixture::new(kind, origin_url.clone()); + let decoded = + fetch_direct_decoded(origin_url, kind, None).await.expect("decoder setup succeeds before streaming"); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + assert_eq!(fixture.segment_failure_count().await, 0, "{kind:?} must not affect segment failure state"); + } + } + + #[tokio::test] + async fn direct_stream_decoder_failure_aborts_body_without_origin_retry() { + let mut truncated = gzip_encode(b"decoder failure after response headers").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("decoder setup succeeds before streaming"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + assert_eq!(origin.requests.lock().await.len(), 1, "streaming errors must not trigger a hidden retry"); + } + + #[tokio::test] + async fn direct_stream_refreshes_origin_body_idle_deadline_after_each_chunk() { + const BODY_IDLE_TIMEOUT_MS: u64 = 500; + let chunks = [b"slow-".to_vec(), b"but-".to_vec(), b"continuous-".to_vec(), b"body".to_vec()]; + let expected = chunks.concat(); + let origin = spawn_test_origin_in_chunks( + "200 OK", + vec![("Content-Type", "application/octet-stream")], + chunks.into(), + Duration::from_millis(200), + ) + .await; + let decoded = fetch_direct_decoded_with_timeout( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + None, + BODY_IDLE_TIMEOUT_MS, + ) + .await + .expect("direct response setup succeeds"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("progressing body outlives total timeout"), + expected + ); + assert_eq!(origin.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn cacheable_transient_rejects_identity_partial_response_before_commit_and_cleans_guard() { + let origin = spawn_test_origin( + "206 Partial Content", + vec![("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], + b"part".to_vec(), + ) + .await; + let fixture = TestTransientCacheFixture::new(format!("{}/partial.ts", origin.base_url)).await; + let dropped_guards = Arc::new(AtomicUsize::new(0)); + let dropped_guards_for_prepare = Arc::clone(&dropped_guards); + + let result = + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), move |_| { + let dropped_guards = Arc::clone(&dropped_guards_for_prepare); + async move { Ok(DropCounter(dropped_guards)) }.boxed() + }) + .await; + + assert!(matches!(result, Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus))); + assert_eq!(dropped_guards.load(Ordering::Relaxed), 1); + assert!(fixture.segment_cache.metadata(&fixture.cache_key).await.expect("cache metadata reads").is_none()); + assert!(!fixture.segment_cache.has_active_temp_files().await); + assert_eq!(std::fs::read_dir(fixture._temp_dir.path()).expect("cache root reads").count(), 0); + let session = fixture.session.read().await; + let entry = session.transient.object_cache.get(&fixture.cache_key).expect("fetching cache entry remains"); + assert!(!matches!(entry.status, TransientObjectCacheStatus::Ready { .. })); + drop(session); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); + } + + #[tokio::test] + async fn cacheable_transient_without_range_decodes_to_identity_before_cache_and_range_reads() { + let ciphertext = + vec![0x00, 0xff, 0x92, 0x10, 0x7e, 0x33, 0xc4, 0x81, 0xa8, 0x09, 0x5d, 0xe2, 0x44, 0x17, 0xb0, 0x6f]; + assert!(std::str::from_utf8(&ciphertext).is_err()); + let encoded = zstd_encode(&ciphertext).await; + assert_ne!(encoded, ciphertext); + let origin = spawn_zstd_origin(encoded).await; + let fixture = TestTransientCacheFixture::new(format!("{}/cipher.ts", origin.base_url)).await; + + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), |_| { + async { Ok(()) }.boxed() + }) + .await + .expect("encoded transient cache fetch succeeds"); + + let metadata = fixture + .segment_cache + .metadata(&fixture.cache_key) + .await + .expect("cache metadata reads") + .expect("decoded object is cached"); + assert_eq!(metadata.size, ciphertext.len() as u64); + assert_eq!(tokio::fs::read(&metadata.path).await.expect("cache body reads"), ciphertext); + let mut range = Vec::new(); + fixture + .segment_cache + .open_range(&fixture.cache_key, 5) + .await + .expect("decoded cache range opens") + .take(4) + .read_to_end(&mut range) + .await + .expect("decoded cache range reads"); + assert_eq!(range, ciphertext[5..9]); + + let session = fixture.session.read().await; + let entry = session.transient.object_cache.get(&fixture.cache_key).expect("transient cache entry"); + assert!(matches!( + entry.status, + TransientObjectCacheStatus::Ready { content_length, .. } + if content_length == ciphertext.len() as u64 + )); + assert_eq!(entry.content_type, "application/octet-stream"); + drop(session); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); + } +} diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index eb9d9e6d8..26e20fc9a 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -4250,6 +4250,7 @@ mod tests { input_name: Arc::from("input"), channel_no: 0, source_ordinal: 0, + input_stream_id: Arc::from("42"), }; InputWorker::apply_pending_generic_probe_base(&batch, XtreamCluster::Video, 42, &mut item); diff --git a/backend/src/api/model/mod.rs b/backend/src/api/model/mod.rs index 436f78e8f..454a37ed8 100644 --- a/backend/src/api/model/mod.rs +++ b/backend/src/api/model/mod.rs @@ -23,7 +23,10 @@ mod streams; mod update_guard; mod xtream; -pub(crate) use self::streams::*; +#[cfg(test)] +pub(in crate::api) use self::hls_provisioning::{ + build_hls_custom_video_manifest_body, hls_panel_provisioning_manifest_path, +}; pub use self::{ active_provider_manager::*, app_state::*, connection_manager::*, event_manager::*, hls_cache::*, hls_provisioning::HlsProvisioningState, metadata_update_manager::*, playlist_mem_cache::*, provider_dns_manager::*, @@ -37,8 +40,7 @@ pub(in crate::api) use self::{ hls_custom_video_manifest_path, hls_custom_video_manifest_response_with_virtual_id, hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, parse_hls_panel_provisioning_segment_route_name, start_hls_panel_provisioning_once, - try_hls_panel_provisioning_manifest_response, HlsPanelProvisioningRedirectPaths, - HlsProvisioningStatus, + try_hls_panel_provisioning_manifest_response, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, }, model_utils::*, provider_config::*, @@ -48,9 +50,9 @@ pub(in crate::api) use self::{ stream_error::*, xtream::*, }; -#[cfg(test)] -pub(in crate::api) use self::hls_provisioning::{ - build_hls_custom_video_manifest_body, hls_panel_provisioning_manifest_path, +pub(crate) use self::{ + hls_cache::{log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource}, + streams::*, }; mod batch_result_collector; pub use self::batch_result_collector::*; diff --git a/backend/src/api/model/model_utils.rs b/backend/src/api/model/model_utils.rs index 599b718a6..21e60237a 100644 --- a/backend/src/api/model/model_utils.rs +++ b/backend/src/api/model/model_utils.rs @@ -1,4 +1,11 @@ -use reqwest::{header::HeaderMap, StatusCode}; +use crate::{ + api::model::{ProviderContentRepresentationMode, ProviderStreamHeader}, + utils::content_coding::parse_content_encoding_tokens, +}; +use reqwest::{ + header::{HeaderMap, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_RANGE, TRANSFER_ENCODING}, + StatusCode, +}; use shared::utils::filter_response_header; use std::{collections::HashSet, str::FromStr}; @@ -10,6 +17,96 @@ pub fn get_response_headers(headers: &HeaderMap) -> Vec<(String, String)> { .collect() } +/// Failures raised while selecting representation-sensitive provider response headers. +#[allow(clippy::enum_variant_names)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub(crate) enum ProviderResponseHeaderError { + #[error("invalid Content-Encoding header")] + InvalidContentEncoding, + + #[error("invalid Content-Length header")] + InvalidContentLength, + + #[error("invalid Content-Range header")] + InvalidContentRange, +} + +/// Selects representation-consistent provider headers without widening the global response allowlist. +pub(crate) fn provider_response_headers( + headers: &HeaderMap, + mode: ProviderContentRepresentationMode, +) -> Result { + let mut selected = headers + .iter() + .filter(|(key, _)| { + let name = key.as_str(); + name != CONTENT_ENCODING.as_str() + && name != CONTENT_LENGTH.as_str() + && name != CONTENT_RANGE.as_str() + && name != TRANSFER_ENCODING.as_str() + }) + .filter(|(key, _)| filter_response_header(key.as_str())) + .filter_map(|(key, value)| value.to_str().ok().map(|value| (key.to_string(), value.to_string()))) + .collect::>(); + + if matches!(mode, ProviderContentRepresentationMode::PreserveOrigin) { + let content_encoding = + parse_content_encoding_tokens(headers).map_err(|_| ProviderResponseHeaderError::InvalidContentEncoding)?; + if !content_encoding.is_empty() { + selected.push((CONTENT_ENCODING.as_str().to_owned(), content_encoding.join(", "))); + } + } + if let Some(content_length) = validated_content_length(headers)? { + selected.push((CONTENT_LENGTH.as_str().to_owned(), content_length)); + } + if let Some(content_range) = validated_content_range(headers)? { + selected.push((CONTENT_RANGE.as_str().to_owned(), content_range)); + } + + Ok(selected) +} + +fn validated_content_length(headers: &HeaderMap) -> Result, ProviderResponseHeaderError> { + let mut content_length = None; + + for value in headers.get_all(CONTENT_LENGTH) { + let value = value.to_str().map_err(|_| ProviderResponseHeaderError::InvalidContentLength)?; + for value in value.split(',') { + let value = value.trim_matches(|character| matches!(character, ' ' | '\t')); + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProviderResponseHeaderError::InvalidContentLength); + } + let value = value.parse::().map_err(|_| ProviderResponseHeaderError::InvalidContentLength)?; + match content_length { + Some(expected) if value != expected => { + return Err(ProviderResponseHeaderError::InvalidContentLength); + } + Some(_) => {} + None => content_length = Some(value), + } + } + } + + Ok(content_length.map(|value| value.to_string())) +} + +fn validated_content_range(headers: &HeaderMap) -> Result, ProviderResponseHeaderError> { + let mut values = headers.get_all(CONTENT_RANGE).iter(); + let Some(value) = values.next() else { + return Ok(None); + }; + if values.next().is_some() { + return Err(ProviderResponseHeaderError::InvalidContentRange); + } + + let value = value.to_str().map_err(|_| ProviderResponseHeaderError::InvalidContentRange)?; + let value = value.trim_matches(|character| matches!(character, ' ' | '\t')); + if value.is_empty() || value.contains(',') { + return Err(ProviderResponseHeaderError::InvalidContentRange); + } + Ok(Some(value.to_owned())) +} + pub fn get_stream_response_with_headers( custom: Option<(Vec<(String, String)>, StatusCode)>, ) -> (axum::http::StatusCode, axum::http::HeaderMap) { @@ -20,6 +117,9 @@ pub fn get_stream_response_with_headers( if let Some((custom_headers, status_code)) = custom { status = status_code; for (key, value) in custom_headers { + if key.eq_ignore_ascii_case(TRANSFER_ENCODING.as_str()) { + continue; + } if let (Ok(name), Ok(val)) = (axum::http::HeaderName::from_str(&key), axum::http::HeaderValue::from_str(&value)) { @@ -47,3 +147,222 @@ pub fn get_stream_response_with_headers( (status, headers) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::content_coding::normalize_headers_after_content_decoding; + use reqwest::header::{ + ACCEPT_RANGES, AUTHORIZATION, CACHE_CONTROL, CONTENT_TYPE, ETAG, PROXY_AUTHENTICATE, SET_COOKIE, + WWW_AUTHENTICATE, + }; + + fn selected_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers.iter().find_map(|(key, value)| key.eq_ignore_ascii_case(name).then_some(value.as_str())) + } + + #[test] + fn provider_preserve_origin_keeps_representation_headers_and_drops_unsafe_headers() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_ENCODING, "gzip".parse().unwrap()); + headers.append(CONTENT_ENCODING, "br".parse().unwrap()); + headers.insert(CONTENT_LENGTH, "321".parse().unwrap()); + headers.insert(CONTENT_RANGE, "bytes 10-330/1000".parse().unwrap()); + headers.insert(TRANSFER_ENCODING, "chunked".parse().unwrap()); + headers.insert(SET_COOKIE, "provider_session=secret".parse().unwrap()); + headers.insert(AUTHORIZATION, "Bearer secret".parse().unwrap()); + headers.insert("x-provider-secret", "secret".parse().unwrap()); + + let selected = provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin) + .expect("valid provider headers"); + + assert_eq!(selected_value(&selected, CONTENT_ENCODING.as_str()), Some("gzip, br")); + assert_eq!(selected_value(&selected, CONTENT_LENGTH.as_str()), Some("321")); + assert_eq!(selected_value(&selected, CONTENT_RANGE.as_str()), Some("bytes 10-330/1000")); + for rejected in [TRANSFER_ENCODING.as_str(), SET_COOKIE.as_str(), AUTHORIZATION.as_str(), "x-provider-secret"] { + assert_eq!(selected_value(&selected, rejected), None, "unexpected header {rejected}"); + } + } + + #[test] + fn provider_preserve_origin_keeps_unknown_valid_content_coding() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_ENCODING, "gzip, X-Provider-Coding".parse().unwrap()); + headers.append(CONTENT_ENCODING, "compress".parse().unwrap()); + + let selected = provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin) + .expect("valid provider headers"); + + assert_eq!(selected_value(&selected, CONTENT_ENCODING.as_str()), Some("gzip, X-Provider-Coding, compress")); + } + + #[test] + fn provider_preserve_origin_rejects_malformed_or_non_text_content_encoding() { + for value in [ + "gzip,,br".parse().unwrap(), + reqwest::header::HeaderValue::from_bytes(&[0xff]).expect("opaque header value"), + ] { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_ENCODING, value); + + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentEncoding) + ); + } + } + + #[test] + fn provider_preserve_origin_canonicalizes_identical_content_lengths() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_LENGTH, "00321".parse().unwrap()); + headers.append(CONTENT_LENGTH, "321, 321".parse().unwrap()); + + let selected = provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin) + .expect("matching content lengths"); + + assert_eq!(selected_value(&selected, CONTENT_LENGTH.as_str()), Some("321")); + } + + #[test] + fn provider_preserve_origin_rejects_invalid_or_conflicting_content_lengths() { + for values in [&["321", "322"][..], &["32x"][..], &[""][..]] { + let mut headers = HeaderMap::new(); + for value in values { + headers.append(CONTENT_LENGTH, (*value).parse().unwrap()); + } + + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentLength) + ); + } + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, reqwest::header::HeaderValue::from_bytes(&[0xff]).expect("opaque header value")); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentLength) + ); + } + + #[test] + fn provider_preserve_origin_rejects_ambiguous_or_non_text_content_range() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_RANGE, "bytes 0-9/20".parse().unwrap()); + headers.append(CONTENT_RANGE, "bytes 10-19/20".parse().unwrap()); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentRange) + ); + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_RANGE, reqwest::header::HeaderValue::from_bytes(&[0xff]).expect("opaque header value")); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentRange) + ); + + for value in ["", "bytes 0-9/20, bytes 10-19/20"] { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_RANGE, value.parse().unwrap()); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::PreserveOrigin), + Err(ProviderResponseHeaderError::InvalidContentRange) + ); + } + } + + #[test] + fn provider_identity_keeps_untransformed_content_length_and_content_range() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, "10".parse().unwrap()); + headers.insert(CONTENT_RANGE, "bytes 10-19/100".parse().unwrap()); + headers.insert(TRANSFER_ENCODING, "chunked".parse().unwrap()); + + let selected = provider_response_headers(&headers, ProviderContentRepresentationMode::Identity) + .expect("valid untransformed identity headers"); + + assert_eq!(selected_value(&selected, CONTENT_LENGTH.as_str()), Some("10")); + assert_eq!(selected_value(&selected, CONTENT_RANGE.as_str()), Some("bytes 10-19/100")); + assert_eq!(selected_value(&selected, CONTENT_ENCODING.as_str()), None); + assert_eq!(selected_value(&selected, TRANSFER_ENCODING.as_str()), None); + } + + #[test] + fn provider_identity_rejects_invalid_untransformed_length_and_range() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, "32x".parse().unwrap()); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::Identity), + Err(ProviderResponseHeaderError::InvalidContentLength) + ); + + let mut headers = HeaderMap::new(); + headers.append(CONTENT_RANGE, "bytes 0-9/20".parse().unwrap()); + headers.append(CONTENT_RANGE, "bytes 10-19/20".parse().unwrap()); + assert_eq!( + provider_response_headers(&headers, ProviderContentRepresentationMode::Identity), + Err(ProviderResponseHeaderError::InvalidContentRange) + ); + } + + #[test] + fn provider_identity_uses_only_safe_normalized_headers() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_ENCODING, "zstd".parse().unwrap()); + headers.insert(CONTENT_LENGTH, "99".parse().unwrap()); + headers.insert(CONTENT_RANGE, "bytes 0-98/99".parse().unwrap()); + headers.insert(ACCEPT_RANGES, "bytes".parse().unwrap()); + headers.insert(ETAG, "\"encoded\"".parse().unwrap()); + headers.insert(TRANSFER_ENCODING, "chunked".parse().unwrap()); + headers.insert(CONTENT_TYPE, "video/mp2t".parse().unwrap()); + headers.insert(CACHE_CONTROL, "public, max-age=30".parse().unwrap()); + headers.insert(SET_COOKIE, "provider_session=secret".parse().unwrap()); + headers.insert(PROXY_AUTHENTICATE, "Basic realm=provider".parse().unwrap()); + headers.insert(WWW_AUTHENTICATE, "Basic realm=provider".parse().unwrap()); + headers.insert("x-provider-secret", "secret".parse().unwrap()); + normalize_headers_after_content_decoding(&mut headers); + + let selected = + provider_response_headers(&headers, ProviderContentRepresentationMode::for_playback_extension(".m3u8")) + .expect("valid normalized provider headers"); + + assert_eq!(selected_value(&selected, CONTENT_TYPE.as_str()), Some("video/mp2t")); + assert_eq!(selected_value(&selected, CACHE_CONTROL.as_str()), Some("public, max-age=30")); + for rejected in [ + CONTENT_ENCODING.as_str(), + CONTENT_LENGTH.as_str(), + CONTENT_RANGE.as_str(), + ACCEPT_RANGES.as_str(), + ETAG.as_str(), + TRANSFER_ENCODING.as_str(), + SET_COOKIE.as_str(), + PROXY_AUTHENTICATE.as_str(), + WWW_AUTHENTICATE.as_str(), + "x-provider-secret", + ] { + assert_eq!(selected_value(&selected, rejected), None, "unexpected header {rejected}"); + } + } + + #[test] + fn rebuilt_stream_responses_never_forward_transfer_encoding() { + let mut origin_headers = HeaderMap::new(); + origin_headers.insert(TRANSFER_ENCODING, "chunked".parse().unwrap()); + origin_headers.insert(CONTENT_TYPE, "video/mp2t".parse().unwrap()); + + let selected = get_response_headers(&origin_headers); + assert_eq!(selected_value(&selected, TRANSFER_ENCODING.as_str()), None); + + let (_, rebuilt) = get_stream_response_with_headers(Some(( + vec![ + (TRANSFER_ENCODING.as_str().to_owned(), "chunked".to_owned()), + (CONTENT_TYPE.as_str().to_owned(), "video/mp2t".to_owned()), + ], + StatusCode::OK, + ))); + assert!(!rebuilt.contains_key(TRANSFER_ENCODING)); + assert_eq!(rebuilt[CONTENT_TYPE], "video/mp2t"); + } +} diff --git a/backend/src/api/model/stream.rs b/backend/src/api/model/stream.rs index 2c97bfd17..4bb935560 100644 --- a/backend/src/api/model/stream.rs +++ b/backend/src/api/model/stream.rs @@ -5,7 +5,10 @@ use crate::{ use axum::http::StatusCode; use bytes::Bytes; use futures::stream::BoxStream; -use shared::model::{PlaylistItemType, StreamChannel}; +use shared::{ + defaults::HLS_EXT, + model::{PlaylistItemType, StreamChannel}, +}; use std::{collections::HashMap, sync::Arc}; use tokio_util::sync::CancellationToken; use url::Url; @@ -18,6 +21,24 @@ pub type ProviderStreamResponse = (Option, ProviderStreamIn pub type ProviderStreamFactoryResponse = (BoxedProviderStream, ProviderStreamInfo); +/// Controls whether a provider stream preserves its origin representation or normalizes it to identity bytes. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) enum ProviderContentRepresentationMode { + #[default] + PreserveOrigin, + Identity, +} + +impl ProviderContentRepresentationMode { + pub(crate) fn for_playback_extension(extension: &str) -> Self { + if extension.eq_ignore_ascii_case(HLS_EXT) { + Self::Identity + } else { + Self::PreserveOrigin + } + } +} + pub(crate) fn uses_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { !stream_channel.shared && matches!( @@ -39,10 +60,7 @@ pub enum ProviderStreamCustomReason { } pub enum ProviderStreamState { - Custom { - response: ProviderStreamResponse, - reason: ProviderStreamCustomReason, - }, + Custom { response: ProviderStreamResponse, reason: ProviderStreamCustomReason }, Available(Option, StreamUrl), GracePeriod(Option, StreamUrl), } @@ -58,6 +76,7 @@ pub struct StreamDetails { pub disable_provider_grace: bool, pub reconnect_flag: Option, pub provider_handle: Option, + pub(crate) content_representation: ProviderContentRepresentationMode, /// Set when the stream was admitted via a user-grace strategy. Carried through to /// `stream_grace_period` so remaining strategies can be evaluated if the grace fails. pub(crate) grace_resolution_context: Option, @@ -79,6 +98,7 @@ impl Clone for StreamDetails { disable_provider_grace: self.disable_provider_grace, reconnect_flag: self.reconnect_flag.clone(), provider_handle: self.provider_handle.clone(), + content_representation: self.content_representation, grace_resolution_context: self.grace_resolution_context.clone(), } } @@ -97,6 +117,7 @@ impl StreamDetails { disable_provider_grace: false, reconnect_flag: None, provider_handle: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, } } @@ -122,3 +143,26 @@ pub struct StreamingStrategy { pub provider_stream_state: ProviderStreamState, pub input_headers: Option>, } + +#[cfg(test)] +mod tests { + use super::ProviderContentRepresentationMode; + + #[test] + fn provider_representation_mode_uses_hls_extension_not_playlist_item_type() { + assert_eq!( + ProviderContentRepresentationMode::for_playback_extension(".m3u8"), + ProviderContentRepresentationMode::Identity + ); + assert_eq!( + ProviderContentRepresentationMode::for_playback_extension(".M3U8"), + ProviderContentRepresentationMode::Identity + ); + for extension in [".ts", ".mp4", ".mkv", ""] { + assert_eq!( + ProviderContentRepresentationMode::for_playback_extension(extension), + ProviderContentRepresentationMode::PreserveOrigin + ); + } + } +} diff --git a/backend/src/api/model/stream_error.rs b/backend/src/api/model/stream_error.rs index dcbd80fd8..47a06f093 100644 --- a/backend/src/api/model/stream_error.rs +++ b/backend/src/api/model/stream_error.rs @@ -5,6 +5,7 @@ use tokio_stream::wrappers::errors::BroadcastStreamRecvError; pub enum StreamError { Reqwest { message: String, class: &'static str, status: Option }, StdIo(String), + ContentDecoding(String), // ReceiverClosed, ReceiverError(BroadcastStreamRecvError), LockError(String), @@ -81,6 +82,7 @@ impl StreamError { match self { Self::Reqwest { class, .. } => class, Self::StdIo(_) => "io", + Self::ContentDecoding(_) => "content_decoding", Self::ReceiverError(_) => "receiver", Self::LockError(_) => "lock", Self::Stream(_) => "stream", @@ -105,6 +107,7 @@ impl std::fmt::Display for StreamError { match self { StreamError::Reqwest { message, .. } => write!(f, "Reqwest error: {message}"), StreamError::StdIo(e) => write!(f, "IO error: {e}"), + StreamError::ContentDecoding(e) => write!(f, "Content decoding error: {e}"), // StreamError::ReceiverClosed => write!(f, "Receiver closed"), StreamError::ReceiverError(e) => write!(f, "Receiver error {e}"), StreamError::Stream(e) => write!(f, "Stream: {e}"), diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index b1c735374..a47f73cf9 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -2,11 +2,11 @@ use crate::{ api::{ api_utils::get_stream_options, model::{ - AppState, BoxedProviderStream, CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, - MeteringStream, PendingProviderWakeSource, ProviderHandle, ProviderStreamFactoryOptions, StreamDetails, - StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer, connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, - create_provider_stream, uses_direct_body_idle_timeout, + create_provider_stream, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, CleanupEvent, + ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, PendingProviderWakeSource, + ProviderHandle, ProviderStreamFactoryOptions, StreamDetails, StreamError, StreamMeterHandle, + TimedClientStream, TransportStreamBuffer, }, panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, }, @@ -14,22 +14,20 @@ use crate::{ model::{ConfigInput, ProxyUserCredentials}, utils::debug_if_enabled, }; -use axum::http::{HeaderMap, header::USER_AGENT}; +use axum::http::{header::USER_AGENT, HeaderMap}; use bytes::Bytes; -use futures::{Future, Stream, StreamExt, task::AtomicWaker}; +use futures::{task::AtomicWaker, Future, Stream, StreamExt}; use log::{error, info, warn}; -use shared::model::FailureStage; -use shared::utils::Internable; use shared::{ - model::{StreamChannel, UserConnectionPermission, VirtualId}, - utils::sanitize_sensitive_info, + model::{FailureStage, StreamChannel, UserConnectionPermission, VirtualId}, + utils::{sanitize_sensitive_info, Internable}, }; use std::{ net::SocketAddr, pin::Pin, sync::{ - Arc, atomic::{AtomicU8, Ordering}, + Arc, }, task::{Context, Poll}, }; @@ -170,9 +168,7 @@ impl DirectBodyIdleTimeout { Self { enabled: false, deadline: None, sleep: None, last_socket_activity_touch: None } } - const fn enabled() -> Self { - Self { enabled: true, deadline: None, sleep: None, last_socket_activity_touch: None } - } + const fn enabled() -> Self { Self { enabled: true, deadline: None, sleep: None, last_socket_activity_touch: None } } fn mark_progress(&mut self) -> bool { if !self.enabled { @@ -528,7 +524,11 @@ fn wrap_timed_client_stream_if_needed( None => stream, Some(mins) => { let secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); - if secs > 0 { TimedClientStream::new(app_state, stream, secs, addr, virtual_id).boxed() } else { stream } + if secs > 0 { + TimedClientStream::new(app_state, stream, secs, addr, virtual_id).boxed() + } else { + stream + } } } } @@ -567,7 +567,9 @@ fn create_deferred_provider_open_future( client_ip: Some(&fingerprint.client_ip), stream_channel: Some(stream_channel), connect_failure_stage: Some(FailureStage::ProviderOpen), - }); + content_representation: stream_details.content_representation, + }) + .for_deferred_open(); provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref())); Some(DeferredProviderOpenState::Pending(Box::new(DeferredProviderOpenContext { @@ -797,9 +799,7 @@ impl Stream for ActiveClientStream { } impl Drop for ActiveClientStream { - fn drop(&mut self) { - self.state.release_stream_and_provider_handle_once(); - } + fn drop(&mut self) { self.state.release_stream_and_provider_handle_once(); } } #[allow(clippy::too_many_lines)] @@ -1334,18 +1334,20 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op #[cfg(test)] mod tests { use super::{ - ActiveClientStream, ActiveClientStreamParams, ActiveClientStreamState, CustomVideoBuffers, - DIRECT_BODY_IDLE_TIMEOUT_SECS, DeferredProviderOpenOutcome, DeferredProviderOpenState, DirectBodyIdleTimeout, - GracePeriodParams, StreamMode, TimedStreamContext, create_active_client_stream, - should_use_direct_body_idle_timeout, stream_grace_period, + create_active_client_stream, create_deferred_provider_open_future, should_use_direct_body_idle_timeout, + stream_grace_period, ActiveClientStream, ActiveClientStreamParams, ActiveClientStreamState, CustomVideoBuffers, + DeferredProviderOpenOutcome, DeferredProviderOpenState, DirectBodyIdleTimeout, GracePeriodParams, StreamMode, + TimedStreamContext, DIRECT_BODY_IDLE_TIMEOUT_SECS, }; - use crate::api::api_utils::GraceResolutionContext; - use crate::api::model::connection_manager::PROVIDER_END_NOT_SET; use crate::{ - api::model::{ - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, - CreateUserSessionParams, CustomVideoStreamType, DownloadQueue, EventManager, MetadataUpdateManager, - PlaylistStorageState, SharedStreamManager, StreamDetails, StreamError, UpdateGuard, + api::{ + api_utils::GraceResolutionContext, + model::{ + connection_manager::PROVIDER_END_NOT_SET, ActiveProviderManager, ActiveUserManager, AppState, + CancelTokens, ConnectionManager, CreateUserSessionParams, CustomVideoStreamType, DownloadQueue, + EventManager, MetadataUpdateManager, PlaylistStorageState, ProviderContentRepresentationMode, + SharedStreamManager, StreamDetails, StreamError, UpdateGuard, + }, }, auth::Fingerprint, model::{ @@ -1357,7 +1359,7 @@ mod tests { use arc_swap::{ArcSwap, ArcSwapOption}; use axum::http::HeaderMap; use bytes::Bytes; - use futures::{StreamExt, pin_mut}; + use futures::{pin_mut, StreamExt}; use reqwest::Client; use shared::{ model::{ @@ -1369,8 +1371,8 @@ mod tests { use std::{ collections::HashMap, sync::{ - Arc, atomic::{AtomicU8, Ordering}, + Arc, }, time::Duration, }; @@ -1610,15 +1612,12 @@ mod tests { provider_name: Some(Arc::clone(provider_name)), request_url: Some("http://provider-1.example/live/1".intern()), session_headers: None, - grace_period: GracePeriodOptions { - period_millis: 100, - timeout_secs: 0, - hold_stream: true, - }, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, provider_grace_active: true, disable_provider_grace: false, reconnect_flag: None, provider_handle: Some(provider_handle), + content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, } } @@ -1979,6 +1978,49 @@ mod tests { app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; } + #[tokio::test] + async fn deferred_provider_open_forces_identity_when_original_mode_preserves_origin() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let addr = "127.0.0.1:55015".parse().unwrap(); + let provider_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .await + .expect("deferred provider allocation"); + let stream_details = create_deferred_provider_grace_details(&provider_name, provider_handle.clone()); + assert_eq!(stream_details.content_representation, ProviderContentRepresentationMode::PreserveOrigin); + let fingerprint = create_test_fingerprint(addr); + let mut stream_channel = create_test_stream_channel(1, "http://provider-1.example/live/1"); + stream_channel.item_type = PlaylistItemType::Catchup; + + let deferred = create_deferred_provider_open_future( + &app_state, + &stream_details, + &fingerprint, + &stream_channel, + &HeaderMap::new(), + ) + .expect("deferred provider open context"); + + let DeferredProviderOpenState::Pending(context) = deferred else { + panic!("new deferred open must start pending") + }; + assert_eq!( + context.provider_stream_factory_options.content_representation(), + ProviderContentRepresentationMode::Identity + ); + assert!(!context.provider_stream_factory_options.response_head_is_available()); + drop(context); + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + } + #[tokio::test(start_paused = true)] async fn test_active_client_stream_deferred_provider_grace_retains_provider_handle_while_grace_pending() { let app_state = create_test_app_state(); @@ -2472,15 +2514,12 @@ mod tests { provider_name: Some(provider_name), request_url: Some("http://provider-1.example/live/2.ts".intern()), session_headers: None, - grace_period: GracePeriodOptions { - period_millis: 100, - timeout_secs: 0, - hold_stream: true, - }, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, provider_grace_active: false, disable_provider_grace: false, reconnect_flag: None, provider_handle: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: Some(grace_context.clone()), }; diff --git a/backend/src/api/model/streams/provider_stream.rs b/backend/src/api/model/streams/provider_stream.rs index d438ff817..1517c2bff 100644 --- a/backend/src/api/model/streams/provider_stream.rs +++ b/backend/src/api/model/streams/provider_stream.rs @@ -13,8 +13,7 @@ use axum::response::IntoResponse; use log::trace; use reqwest::StatusCode; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use shared::error::TuliproxError; -use shared::model::PlaylistItemType; +use shared::{error::TuliproxError, model::PlaylistItemType}; use std::{fmt, net::SocketAddr, str::FromStr, sync::Arc}; use tokio_util::sync::CancellationToken; @@ -87,7 +86,8 @@ fn prepare_video_headers(headers: &[(String, String)]) -> Vec<(String, String)> || key.eq_ignore_ascii_case("content-length") || key.eq_ignore_ascii_case("range") || key.eq_ignore_ascii_case("content-range") - || key.eq_ignore_ascii_case("accept-ranges")) + || key.eq_ignore_ascii_case("accept-ranges") + || key.eq_ignore_ascii_case("transfer-encoding")) }) .map(|(key, value)| (key.clone(), value.clone())) .collect(); @@ -146,12 +146,11 @@ fn create_video_stream( } if let Some(video) = video_buffer { trace!("{log_message}"); - let stream = - apply_custom_stream_timeout(cfg, Box::pin(ThrottledStream::new(CustomVideoStream::new(video.clone()), 8000))); - ( - Some(stream), - Some((prepare_video_headers(headers), status, None, Some(stream_type))), - ) + let stream = apply_custom_stream_timeout( + cfg, + Box::pin(ThrottledStream::new(CustomVideoStream::new(video.clone()), 8000)), + ); + (Some(stream), Some((prepare_video_headers(headers), status, None, Some(stream_type)))) } else { (None, None) } @@ -190,10 +189,7 @@ pub fn create_channel_unavailable_stream( /// and human-readable description. macro_rules! ok_custom_stream_factory { ($fn_name:ident, $field:ident, $stream_type:expr, $description:expr) => { - pub fn $fn_name( - cfg: &AppConfig, - headers: &[(String, String)], - ) -> ProviderStreamResponse { + pub fn $fn_name(cfg: &AppConfig, headers: &[(String, String)]) -> ProviderStreamResponse { let custom_stream_response = cfg.custom_stream_response.load(); let video = custom_stream_response.as_ref().and_then(|c| c.$field.as_ref()); create_ok_video_stream(cfg, $stream_type, video, headers, $description) @@ -273,9 +269,7 @@ pub fn create_custom_video_stream_response( ) -> impl axum::response::IntoResponse + Send { let config = &app_state.app_config; if let (Some(stream), Some((headers, status_code, _, _))) = match video_response { - CustomVideoStreamType::ChannelUnavailable => { - create_channel_unavailable_stream(config, &[], StatusCode::OK) - } + CustomVideoStreamType::ChannelUnavailable => create_channel_unavailable_stream(config, &[], StatusCode::OK), CustomVideoStreamType::UserConnectionsExhausted => create_user_connections_exhausted_stream(config, &[]), CustomVideoStreamType::ProviderConnectionsExhausted => { create_provider_connections_exhausted_stream(config, &[]) @@ -316,8 +310,8 @@ pub fn get_header_filter_for_item_type(item_type: PlaylistItemType) -> HeaderFil #[cfg(test)] mod tests { use super::{ - create_channel_unavailable_stream, is_custom_video_stream_enabled, get_custom_stream_response_error_status, - CustomVideoStreamType, + create_channel_unavailable_stream, get_custom_stream_response_error_status, is_custom_video_stream_enabled, + prepare_video_headers, CustomVideoStreamType, }; use crate::{ api::model::TransportStreamBuffer, @@ -351,7 +345,10 @@ mod tests { let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; let app_cfg = AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { custom_stream_response_enabled: true, ..Config::default()})), + config: Arc::new(ArcSwap::from_pointee(Config { + custom_stream_response_enabled: true, + ..Config::default() + })), sources: Arc::new(ArcSwap::from_pointee(sources)), hdhomerun: Arc::new(ArcSwapOption::default()), api_proxy: Arc::new(ArcSwapOption::default()), @@ -404,6 +401,17 @@ mod tests { assert_eq!(parsed.to_string(), "hls_session_or_lease_expired"); } + #[test] + fn custom_video_headers_drop_transfer_encoding() { + let headers = prepare_video_headers(&[ + ("Transfer-Encoding".to_string(), "chunked".to_string()), + ("cache-control".to_string(), "no-store".to_string()), + ]); + + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("transfer-encoding"))); + assert!(headers.iter().any(|(name, value)| name == "cache-control" && value == "no-store")); + } + #[tokio::test] async fn test_channel_unavailable_preserves_supplied_status_code() { let app_cfg = create_test_app_config_with_channel_unavailable(); diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/src/api/model/streams/provider_stream_factory.rs index 1d7fb7405..0138b8bbc 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/src/api/model/streams/provider_stream_factory.rs @@ -1,48 +1,69 @@ use crate::{ - api::{ - api_utils::{get_headers_from_request, StreamOptions}, - model::{ - create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, + api::{ + api_utils::{get_headers_from_request, StreamOptions}, + model::{ + create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, + log_hls_origin_content_coding, + model_utils::{provider_response_headers, ProviderResponseHeaderError}, streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, - AppState, CustomVideoStreamType, ProviderStreamFactoryResponse, StreamError, + AppState, CustomVideoStreamType, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, + ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, }, }, - model::{ConfigProvider, ReverseProxyDisabledHeaderConfig}, + model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, utils::{ + content_coding::{ + apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, + ContentCodingDetection, ContentCodingError, OutboundContentCodingPolicy, + }, debug_if_enabled, request::{ - get_request_headers, preview_request_diagnostics_for_logging, preview_request_target_for_logging, - send_with_retry_and_provider_policy, + get_request_headers, is_safe_cross_origin_redirect_header, preview_request_diagnostics_for_logging, + preview_request_target_for_logging, send_with_retry_and_provider_policy, }, }, }; use futures::{StreamExt, TryStreamExt}; use log::{debug, log_enabled, warn}; use reqwest::{ - header::{HeaderMap, RANGE}, + header::{HeaderMap, CONTENT_RANGE, RANGE}, StatusCode, }; use shared::{ create_bitset, - model::{PlaylistItemType, StreamChannel, StreamInfo}, - utils::{filter_request_header, is_sanitize_sensitive_info_enabled, sanitize_sensitive_info}, + defaults::DEFAULT_USER_AGENT, + model::{ConnectFailureReason, FailureStage, PlaylistItemType, StreamChannel, StreamInfo}, + utils::{filter_request_header, is_sanitize_sensitive_info_enabled, sanitize_sensitive_info, Internable}, }; use std::{ collections::HashMap, + error::Error as StdError, + io, net::SocketAddr, sync::Arc, time::{Duration, Instant}, }; -use tokio_util::sync::CancellationToken; +use tokio_util::{io::ReaderStream, sync::CancellationToken}; use url::Url; -use shared::{ - model::{ConnectFailureReason, FailureStage}, - utils::Internable, - defaults::DEFAULT_USER_AGENT}; const RETRY_SECONDS: u64 = 5; const ERR_MAX_RETRY_COUNT: u32 = 5; +/// Describes whether the provider response head is available before Tuliprox commits the client response head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ProviderResponseHeadAvailability { + Available, + Unavailable, +} + +#[derive(Clone, Copy, Debug)] +struct ProviderStreamPreparationContext { + representation: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, + range_requested: bool, + hls_content_coding_object_kind: Option, +} + create_bitset!( u8, ProviderStreamFactoryFlags, @@ -71,6 +92,9 @@ pub struct ProviderStreamFactoryOptions { user_agent: Option, stream_channel: Option, connect_failure_stage: Option, + content_representation: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, + hls_content_coding_object_kind: Option, } pub(crate) struct ProviderStreamFactoryParams<'a> { @@ -88,6 +112,7 @@ pub(crate) struct ProviderStreamFactoryParams<'a> { pub client_ip: Option<&'a str>, pub stream_channel: Option<&'a StreamChannel>, pub connect_failure_stage: Option, + pub content_representation: ProviderContentRepresentationMode, } impl ProviderStreamFactoryOptions { @@ -107,6 +132,7 @@ impl ProviderStreamFactoryOptions { client_ip, stream_channel, connect_failure_stage, + content_representation, } = request; let buffer_size = if stream_options.buffer_enabled { stream_options.buffer_size } else { 0 }; let user_agent = req_headers @@ -176,6 +202,12 @@ impl ProviderStreamFactoryOptions { user_agent, stream_channel: stream_channel.cloned(), connect_failure_stage: *connect_failure_stage, + content_representation: *content_representation, + response_head_availability: ProviderResponseHeadAvailability::Available, + hls_content_coding_object_kind: match *content_representation { + ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other), + ProviderContentRepresentationMode::PreserveOrigin => None, + }, } } @@ -208,7 +240,9 @@ impl ProviderStreamFactoryOptions { fn get_item_type(&self) -> PlaylistItemType { self.item_type } #[inline] - pub fn should_retry_provider_request(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::RetryEnabled) } + pub fn should_retry_provider_request(&self) -> bool { + self.flags.contains(ProviderStreamFactoryFlags::RetryEnabled) + } #[inline] pub fn should_retry_initial_open_loop(&self) -> bool { @@ -255,6 +289,25 @@ impl ProviderStreamFactoryOptions { fn get_connect_failure_stage(&self) -> Option { self.connect_failure_stage } + #[inline] + pub(crate) fn content_representation(&self) -> ProviderContentRepresentationMode { self.content_representation } + + /// Converts an open delayed until body polling to the only representation safe without an origin response head. + pub(crate) fn for_deferred_open(mut self) -> Self { + self.content_representation = ProviderContentRepresentationMode::Identity; + self.response_head_availability = ProviderResponseHeadAvailability::Unavailable; + self + } + + #[cfg(test)] + pub(crate) fn response_head_is_available(&self) -> bool { + matches!(self.response_head_availability, ProviderResponseHeadAvailability::Available) + } + + #[cfg(test)] + pub(crate) fn hls_content_coding_object_kind(&self) -> Option { + self.hls_content_coding_object_kind + } } fn merge_provider_request_headers( @@ -282,26 +335,20 @@ fn record_provider_open_failure( provider_error_class: Option<&str>, ) { let Some(failure_stage) = stream_options.get_connect_failure_stage() else { return }; - let provider_name = stream_options - .get_provider() - .map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); + let provider_name = + stream_options.get_provider().map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return }; // Resolve target_name from target_id using the stable target config name. - let target_name = app_state - .app_config - .get_target_by_id(info.channel.target_id) - .as_deref() - .map(|t| (&t.name).intern()); - app_state - .connection_manager - .record_connect_failed_with_provider_failure( - &info, - reason, - failure_stage, - provider_http_status.map(|status| status.as_u16()), - provider_error_class, - target_name, - ); + let target_name = + app_state.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); + app_state.connection_manager.record_connect_failed_with_provider_failure( + &info, + reason, + failure_stage, + provider_http_status.map(|status| status.as_u16()), + provider_error_class, + target_name, + ); } fn classify_provider_status_error(status: StatusCode) -> &'static str { @@ -329,11 +376,34 @@ fn should_reject_success_response_content_type(item_type: PlaylistItemType, head #[derive(Clone, Copy, Debug)] enum ProviderStreamRequestFailure { - Status { - status: StatusCode, - provider_error_class: &'static str, - serve_channel_unavailable: bool, - }, + Status { status: StatusCode, provider_error_class: &'static str, serve_channel_unavailable: bool }, +} + +#[derive(Debug, thiserror::Error)] +enum ProviderStreamPreparationError { + #[error(transparent)] + ContentCoding(#[from] ContentCodingError), + + #[error(transparent)] + ResponseHeader(#[from] ProviderResponseHeaderError), + + #[error("provider response head is unavailable for status {status} or Content-Range={has_content_range}")] + DeferredResponseHead { status: StatusCode, has_content_range: bool }, +} + +impl ProviderStreamPreparationError { + fn provider_error_class(&self) -> &'static str { + match self { + Self::ContentCoding(ContentCodingError::InvalidHeader | ContentCodingError::Unsupported(_)) + | Self::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding) => "content_encoding", + Self::ContentCoding(ContentCodingError::EncodedPartialContent) => "encoded_partial_content", + Self::ContentCoding(ContentCodingError::PrefixRead(_)) => "body", + Self::ResponseHeader( + ProviderResponseHeaderError::InvalidContentLength | ProviderResponseHeaderError::InvalidContentRange, + ) + | Self::DeferredResponseHead { .. } => "response_headers", + } + } } impl ProviderStreamRequestFailure { @@ -361,9 +431,10 @@ fn classify_provider_io_error(err: &std::io::Error) -> &'static str { match err.kind() { ErrorKind::TimedOut => "timeout", - ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset | ErrorKind::ConnectionAborted | ErrorKind::NotConnected => { - "connect" - } + ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::NotConnected => "connect", ErrorKind::AddrNotAvailable => "dns", _ => { let lowered = err.to_string().to_ascii_lowercase(); @@ -412,10 +483,25 @@ fn get_request_range_start_bytes(req_headers: &HashMap>) -> Opti // } // } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ProviderRequestCredentialState { + OriginalOrigin, + Scrubbed, +} + +impl ProviderRequestCredentialState { + fn observe_target(&mut self, original_url: &Url, target_url: &Url) { + if !same_origin(original_url, target_url) { + *self = Self::Scrubbed; + } + } +} + fn prepare_client( request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, url_override: Option<&Url>, + credential_state: ProviderRequestCredentialState, ) -> (reqwest::RequestBuilder, bool) { let original_url = stream_options.get_url(); let url = url_override.unwrap_or(original_url); @@ -435,9 +521,16 @@ fn prepare_client( } } - remove_sensitive_headers_on_cross_origin(&mut headers, original_url, url_override); + if matches!(credential_state, ProviderRequestCredentialState::Scrubbed) { + remove_sensitive_headers(&mut headers); + } prepare_default_headers(&mut headers, stream_options); let partial = prepare_partial_request_headers(&mut headers, stream_options, range_start); + let content_coding_policy = match stream_options.content_representation() { + ProviderContentRepresentationMode::PreserveOrigin => OutboundContentCodingPolicy::Inherit, + ProviderContentRepresentationMode::Identity => OutboundContentCodingPolicy::Identity, + }; + apply_outbound_content_coding_policy(&mut headers, content_coding_policy); if log_enabled!(log::Level::Debug) { let message = format!( @@ -452,25 +545,25 @@ fn prepare_client( (request_builder, partial) } -fn remove_sensitive_headers_on_cross_origin( - headers: &mut axum::http::HeaderMap, - original_url: &reqwest::Url, - url_override: Option<&reqwest::Url>, -) { - let Some(override_url) = url_override else { - return; - }; - - let cross_origin = override_url.scheme() != original_url.scheme() - || override_url.host_str() != original_url.host_str() - || override_url.port_or_known_default() != original_url.port_or_known_default(); - - if !cross_origin { - return; +fn remove_sensitive_headers(headers: &mut axum::http::HeaderMap) { + let names_to_remove = headers + .keys() + .filter(|name| !is_safe_cross_origin_redirect_header(name.as_str())) + .cloned() + .collect::>(); + for name in names_to_remove { + headers.remove(name); } +} - headers.remove(axum::http::header::AUTHORIZATION); - headers.remove(axum::http::header::COOKIE); +fn provider_headers_require_manual_redirects(headers: &HeaderMap) -> bool { + headers.keys().any(|name| !is_safe_cross_origin_redirect_header(name.as_str())) +} + +fn same_origin(lhs: &Url, rhs: &Url) -> bool { + lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) + && lhs.host_str() == rhs.host_str() + && lhs.port_or_known_default() == rhs.port_or_known_default() } fn prepare_default_headers(headers: &mut axum::http::HeaderMap, stream_options: &ProviderStreamFactoryOptions) { @@ -527,20 +620,21 @@ fn collect_debug_headers(headers: &HeaderMap) -> Vec<(String, String)> { async fn send_with_manual_redirects( request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, - app_state: &Arc, + app_config: &Arc, ) -> Result { let mut current_url = stream_options.get_url().clone(); let mut remaining_redirects = 10u8; let provider = stream_options.get_provider().cloned(); + let mut credential_state = ProviderRequestCredentialState::OriginalOrigin; loop { let result = send_with_retry_and_provider_policy( - &app_state.app_config, + app_config, ¤t_url, provider.as_ref(), true, stream_options.should_retry_provider_request(), - |resolved_url| prepare_client(request_client, stream_options, Some(resolved_url)).0, + |resolved_url| prepare_client(request_client, stream_options, Some(resolved_url), credential_state).0, ) .await; @@ -568,10 +662,12 @@ async fn send_with_manual_redirects( let Ok(location_str) = location.to_str() else { return Ok(response); }; - let next_url = current_url.join(location_str).or_else(|_| Url::parse(location_str)); + let response_url = response.url().clone(); + let next_url = response_url.join(location_str).or_else(|_| Url::parse(location_str)); let Ok(next_url) = next_url else { return Ok(response); }; + credential_state.observe_target(&response_url, &next_url); current_url = next_url; remaining_redirects = remaining_redirects.saturating_sub(1); continue; @@ -580,23 +676,159 @@ async fn send_with_manual_redirects( } } +#[cfg(test)] +async fn prepare_provider_stream_response( + response: reqwest::Response, + mode: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, +) -> Result { + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: mode, + response_head_availability, + range_requested: false, + hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) + .then_some(HlsOriginContentCodingObjectKind::Other), + }, + Duration::from_secs(STREAM_IDLE_TIMEOUT), + ) + .await +} + +#[cfg(test)] +async fn prepare_provider_stream_response_with_idle_timeout( + response: reqwest::Response, + mode: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, + idle_timeout: Duration, +) -> Result { + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: mode, + response_head_availability, + range_requested: false, + hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) + .then_some(HlsOriginContentCodingObjectKind::Other), + }, + idle_timeout, + ) + .await +} + +async fn prepare_provider_stream_response_for_request( + response: reqwest::Response, + stream_options: &ProviderStreamFactoryOptions, +) -> Result { + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: stream_options.content_representation(), + response_head_availability: stream_options.response_head_availability, + range_requested: stream_options.was_range_requested(), + hls_content_coding_object_kind: stream_options.hls_content_coding_object_kind, + }, + Duration::from_secs(STREAM_IDLE_TIMEOUT), + ) + .await +} + +async fn prepare_provider_stream_response_with_context( + response: reqwest::Response, + context: ProviderStreamPreparationContext, + idle_timeout: Duration, +) -> Result { + match context.representation { + ProviderContentRepresentationMode::PreserveOrigin => { + if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) { + return Err(ProviderStreamPreparationError::DeferredResponseHead { + status: response.status(), + has_content_range: response.headers().contains_key(CONTENT_RANGE), + }); + } + let headers = provider_response_headers(response.headers(), context.representation)?; + let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); + let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); + Ok((stream, response_info)) + } + ProviderContentRepresentationMode::Identity => { + let origin_status = response.status(); + let origin_has_content_range = response.headers().contains_key(CONTENT_RANGE); + // `deflate` decoder selection reads a prefix before the returned stream can enter the + // normal provider body wrappers, so decoder setup must own the same idle bound too. + let decoded = tokio::time::timeout( + idle_timeout, + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly), + ) + .await + .map_err(|_| { + ContentCodingError::PrefixRead(io::Error::new( + io::ErrorKind::TimedOut, + "provider body idle timeout during content-decoder setup", + )) + })??; + if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) + && (origin_status != StatusCode::OK || origin_has_content_range) + { + return Err(ProviderStreamPreparationError::DeferredResponseHead { + status: origin_status, + has_content_range: origin_has_content_range, + }); + } + if let (Some(observation), Some(object_kind)) = + (decoded.content_coding_observation(), context.hls_content_coding_object_kind) + { + log_hls_origin_content_coding( + observation, + object_kind, + context.range_requested, + HlsOriginContentCodingSource::Legacy, + ); + } + let headers = provider_response_headers(&decoded.headers, context.representation)?; + let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); + let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); + Ok((stream, response_info)) + } + } +} + +fn provider_decoded_body_error(error: &io::Error) -> StreamError { + if let Some(error) = content_decoding_error_from_io(error) { + return StreamError::ContentDecoding(format!("coding={}", error.coding.as_http_token())); + } + + let mut source = StdError::source(error); + while let Some(current) = source { + if let Some(error) = current.downcast_ref::() { + return StreamError::reqwest(error); + } + source = current.source(); + } + StreamError::StdIo(error.to_string()) +} + #[allow(clippy::too_many_lines)] async fn provider_stream_request( app_state: &Arc, request_client: &reqwest::Client, stream_options: &ProviderStreamFactoryOptions, ) -> Result, ProviderStreamRequestFailure> { + let use_manual_redirects = app_state.should_use_manual_redirects() + || provider_headers_require_manual_redirects(stream_options.get_headers()); if log_enabled!(log::Level::Debug) { - let diagnostics = preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); debug!( "Provider request diagnostics: manual_redirects={}, {}", - app_state.should_use_manual_redirects(), + use_manual_redirects, sanitize_sensitive_info(&diagnostics) ); } - let response_result = if app_state.should_use_manual_redirects() { + let response_result = if use_manual_redirects { let client_no_redirect = app_state.http_client_no_redirect.load(); - send_with_manual_redirects(&client_no_redirect, stream_options, app_state).await + send_with_manual_redirects(&client_no_redirect, stream_options, &app_state.app_config).await } else { // Use send_with_retry_and_provider for automatic failover support let url = stream_options.get_url(); @@ -609,19 +841,25 @@ async fn provider_stream_request( false, stream_options.should_retry_provider_request(), |resolved_url| { - let (client, _partial_content) = prepare_client(request_client, stream_options, Some(resolved_url)); + let (client, _partial_content) = prepare_client( + request_client, + stream_options, + Some(resolved_url), + ProviderRequestCredentialState::OriginalOrigin, + ); client }, ) .await }; match response_result { - Ok(mut response) => { + Ok(response) => { let status = response.status(); let response_url = response.url().clone(); if log_enabled!(log::Level::Debug) && !status.is_success() { let debug_headers = collect_debug_headers(response.headers()); - let diagnostics = preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); let message = format!( "Provider response error: status={status}, url={response_url}, headers={debug_headers:?}, {diagnostics}" @@ -640,25 +878,24 @@ async fn provider_stream_request( serve_channel_unavailable: true, }); } - let response_info = { + let response = prepare_provider_stream_response_for_request(response, stream_options).await; + let (provider_stream, response_info) = match response { + Ok(response) => response, + Err(error) => { + let provider_error_class = error.provider_error_class(); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class, + serve_channel_unavailable: true, + }); + } + }; + if log_enabled!(log::Level::Debug) { // Unfortunately, the HEAD request does not work, so we need this workaround. // We need some header information from the provider, we extract the necessary headers and forward them to the client - if log_enabled!(log::Level::Debug) { - let message = format!( - "Provider response status: '{}' headers: {:?}", - response.status(), - response.headers_mut() - ); - debug!("{}", sanitize_sensitive_info(&message)); - } - - let response_headers: Vec<(String, String)> = get_response_headers(response.headers()); - //let url = stream_options.get_url(); - // debug!("First headers {headers:?} {} {}", sanitize_sensitive_info(url.as_str())); - Some((response_headers, response.status(), Some(response.url().clone()), None)) - }; - - let provider_stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)).boxed(); + let message = format!("Provider response info: {response_info:?}"); + debug!("{}", sanitize_sensitive_info(&message)); + } return Ok(Some((provider_stream, response_info))); } @@ -670,13 +907,11 @@ async fn provider_stream_request( | StatusCode::UNAUTHORIZED | StatusCode::PROXY_AUTHENTICATION_REQUIRED | StatusCode::METHOD_NOT_ALLOWED - | StatusCode::BAD_REQUEST => { - Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: true, - }) - } + | StatusCode::BAD_REQUEST => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: true, + }), _ => Err(ProviderStreamRequestFailure::Status { status, provider_error_class: classify_provider_status_error(status), @@ -690,13 +925,11 @@ async fn provider_stream_request( StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::GATEWAY_TIMEOUT => { - Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: true, - }) - } + | StatusCode::GATEWAY_TIMEOUT => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: true, + }), _ => Err(ProviderStreamRequestFailure::Status { status, provider_error_class: classify_provider_status_error(status), @@ -711,7 +944,8 @@ async fn provider_stream_request( }) } Err(err) => { - let diagnostics = preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); debug!( "Provider request failed: {}, {}", sanitize_sensitive_info(&err.to_string()), @@ -794,15 +1028,9 @@ async fn get_provider_stream( } connect_err += 1; tokio::time::sleep(Duration::from_millis(50)).await; - debug_if_enabled!( - "Reconnecting stream {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); + debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); } - debug_if_enabled!( - "Stopped reconnecting stream {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); + debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); stream_options.cancel_reconnect(); app_state.connection_manager.release_provider_connection(&stream_options.addr).await; Err(ProviderStreamRequestFailure::Status { @@ -846,7 +1074,10 @@ pub async fn create_provider_stream( } else { stream }; - Some((ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()).boxed(), info)) + Some(( + ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()).boxed(), + info, + )) } Ok(None) => None, Err(failure) => { @@ -874,11 +1105,307 @@ pub async fn create_provider_stream( #[cfg(test)] mod tests { use super::*; + use crate::{ + api::model::BoxedProviderStream, + model::{Config, MediaToolCapabilities, SourcesConfig}, + utils::FileLockManager, + }; + use arc_swap::{ArcSwap, ArcSwapOption}; use axum::http::HeaderMap; + use bytes::Bytes; + use futures::TryStreamExt; use shared::{ - model::{PlaylistItemType, StreamChannel, XtreamCluster}, + model::{ConfigPaths, PlaylistItemType, StreamChannel, XtreamCluster}, utils::Internable, }; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + }; + + struct RedirectRoundTrip { + entry_url: Url, + first_origin_task: tokio::task::JoinHandle>, + second_origin_task: tokio::task::JoinHandle, + } + + fn test_app_config() -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) + } + + async fn read_http_request(socket: &mut TcpStream) -> String { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + String::from_utf8_lossy(&request).into_owned() + } + + async fn spawn_redirect_round_trip(encoded_body: Vec) -> RedirectRoundTrip { + let first_listener = TcpListener::bind("127.0.0.1:0").await.expect("first test origin binds"); + let first_addr = first_listener.local_addr().expect("first test origin address"); + let second_listener = TcpListener::bind("127.0.0.1:0").await.expect("second test origin binds"); + let second_addr = second_listener.local_addr().expect("second test origin address"); + + let first_origin_task = tokio::spawn(async move { + let (mut first_socket, _) = first_listener.accept().await.expect("first origin accepts entry request"); + let entry_request = read_http_request(&mut first_socket).await; + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://localhost:{}/hop\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + second_addr.port() + ); + first_socket.write_all(redirect.as_bytes()).await.expect("first origin redirects to second origin"); + + let (mut final_socket, _) = first_listener.accept().await.expect("first origin accepts final request"); + let final_request = read_http_request(&mut final_socket).await; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded_body.len() + ); + final_socket.write_all(response.as_bytes()).await.expect("first origin writes final response head"); + final_socket.write_all(&encoded_body).await.expect("first origin writes final response body"); + vec![entry_request, final_request] + }); + + let second_origin_task = tokio::spawn(async move { + let (mut socket, _) = second_listener.accept().await.expect("second origin accepts redirect request"); + let request = read_http_request(&mut socket).await; + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{first_addr}/final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + socket.write_all(redirect.as_bytes()).await.expect("second origin redirects to first origin"); + request + }); + + RedirectRoundTrip { + entry_url: Url::parse(&format!("http://{first_addr}/entry")).expect("entry URL"), + first_origin_task, + second_origin_task, + } + } + + fn redirect_test_options(stream_url: &Url) -> ProviderStreamFactoryOptions { + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().expect("Accept-Encoding")); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client-secret".parse().expect("Authorization")); + req_headers.insert(reqwest::header::COOKIE, "session=client-secret".parse().expect("Cookie")); + test_options(ProviderContentRepresentationMode::Identity, stream_url, &req_headers, None, None) + } + + fn assert_redirect_request_headers(request: &str, credentials_expected: bool) { + let request = request.to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); + assert_eq!(request.contains("\r\nauthorization: bearer client-secret\r\n"), credentials_expected); + assert_eq!(request.contains("\r\ncookie: session=client-secret\r\n"), credentials_expected); + } + + async fn assert_identity_redirect_result( + response: reqwest::Response, + first_origin_task: tokio::task::JoinHandle>, + second_origin_task: tokio::task::JoinHandle, + ) { + const IDENTITY_BODY: &[u8] = b"decoded redirect body"; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("redirect response is prepared"); + assert_eq!(collect_stream(stream).await.expect("redirect body decodes"), IDENTITY_BODY); + let (headers, status, _, _) = info.expect("redirect response metadata"); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + + let first_requests = tokio::time::timeout(Duration::from_secs(2), first_origin_task) + .await + .expect("first origin finishes") + .expect("first origin task succeeds"); + let second_request = tokio::time::timeout(Duration::from_secs(2), second_origin_task) + .await + .expect("second origin finishes") + .expect("second origin task succeeds"); + assert_eq!(first_requests.len(), 2); + assert_redirect_request_headers(&first_requests[0], true); + assert_redirect_request_headers(&second_request, false); + assert_redirect_request_headers(&first_requests[1], false); + } + + fn test_options( + mode: ProviderContentRepresentationMode, + stream_url: &Url, + req_headers: &HeaderMap, + input_headers: Option<&HashMap>, + session_headers: Option<&HashMap>, + ) -> ProviderStreamFactoryOptions { + let stream_options = + StreamOptions { stream_retry: true, buffer_enabled: false, buffer_size: 0, pipe_provider_stream: false }; + ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr: "127.0.0.1:8080".parse().unwrap(), + item_type: PlaylistItemType::Catchup, + share_stream: false, + stream_options: &stream_options, + stream_url, + req_headers, + input_headers, + session_headers, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: mode, + }) + } + + async fn local_response( + status: StatusCode, + headers: &[(&str, &str)], + body: Vec, + ) -> (reqwest::Response, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let task_requests = Arc::clone(&requests); + let response_headers = headers.iter().map(|(name, value)| format!("{name}: {value}\r\n")).collect::(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + task_requests.fetch_add(1, Ordering::SeqCst); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let reason = status.canonical_reason().unwrap_or("Status"); + let head = format!( + "HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n{response_headers}Connection: close\r\n\r\n", + status.as_u16(), + body.len() + ); + socket.write_all(head.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + + let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); + (response, requests) + } + + async fn local_stalled_deflate_response() -> (reqwest::Response, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.unwrap(); + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(5)).await; + }); + let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); + (response, task) + } + + #[derive(Clone, Copy)] + enum TestEncoding { + Gzip, + Zlib, + RawDeflate, + Brotli, + Zstd, + } + + async fn encode(body: &[u8], encoding: TestEncoding) -> Vec { + match encoding { + TestEncoding::Gzip => { + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Zlib => { + let mut encoder = async_compression::tokio::write::ZlibEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::RawDeflate => { + let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Brotli => { + let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Zstd => { + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + } + } + + async fn collect_stream(stream: BoxedProviderStream) -> Result, StreamError> { + stream + .try_fold(Vec::new(), |mut bytes, chunk| async move { + bytes.extend_from_slice(&chunk); + Ok(bytes) + }) + .await + } #[test] fn test_provider_stream_factory_options_range_logic() { @@ -905,6 +1432,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); assert_eq!(options.get_range_start_bytes(), Some(0)); // Should track range start even if not requested @@ -926,6 +1454,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(options.was_range_requested()); assert_eq!(options.get_range_start_bytes(), Some(100)); @@ -947,6 +1476,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); assert_eq!(options.get_range_start_bytes(), None); // Should NOT track range start @@ -969,6 +1499,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); // Stripped by filter assert_eq!(options.get_range_start_bytes(), None); @@ -997,6 +1528,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); let dash_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { @@ -1014,6 +1546,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(hls_options.should_retry_provider_request()); @@ -1045,6 +1578,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!( @@ -1092,6 +1626,7 @@ mod tests { epg_reference_ts: None, }), connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); let info = options.build_connect_failed_stream_info("provider-a".intern()).expect("history context"); @@ -1147,6 +1682,7 @@ mod tests { client_ip: None, stream_channel: None, connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert_eq!( @@ -1154,4 +1690,540 @@ mod tests { Some("sid=abc; pref=1") ); } + + #[test] + fn cross_origin_provider_requests_keep_only_redirect_safe_headers() { + let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + req_headers.insert(reqwest::header::USER_AGENT, "test-agent".parse().unwrap()); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); + req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); + req_headers.insert(reqwest::header::PROXY_AUTHORIZATION, "Basic proxy-secret".parse().unwrap()); + let input_headers = HashMap::from([ + ("X-API-Key".to_string(), "api-secret".to_string()), + ("X-Provider-Token".to_string(), "provider-secret".to_string()), + ]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + assert!(provider_headers_require_manual_redirects(options.get_headers())); + + let mut safe_headers = HeaderMap::new(); + safe_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + safe_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + let safe_options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &safe_headers, None, None); + assert!(!provider_headers_require_manual_redirects(safe_options.get_headers())); + + let client = reqwest::Client::new(); + let sensitive_headers = ["authorization", "cookie", "proxy-authorization", "x-api-key", "x-provider-token"]; + + let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); + let same_origin_request = prepare_client( + &client, + &options, + Some(&same_origin), + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .unwrap(); + for name in sensitive_headers { + assert!(same_origin_request.headers().contains_key(name), "same-origin request lost {name}"); + } + + let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); + for (target, credential_state) in [ + (Some(&cross_origin), ProviderRequestCredentialState::Scrubbed), + (None, ProviderRequestCredentialState::Scrubbed), + ] { + let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); + for name in sensitive_headers { + assert!(!request.headers().contains_key(name), "cross-origin request retained {name}"); + } + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-"); + assert_eq!(request.headers()[reqwest::header::USER_AGENT], "test-agent"); + } + } + + #[test] + fn provider_scheme_resolution_keeps_configured_headers_before_redirect() { + let stream_url = Url::parse("provider://mirrors/live/segment.ts").unwrap(); + let req_headers = HeaderMap::new(); + let input_headers = HashMap::from([("X-API-Key".to_string(), "api-secret".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + let resolved_url = Url::parse("https://mirror-a.example/live/segment.ts").unwrap(); + + let request = prepare_client( + &reqwest::Client::new(), + &options, + Some(&resolved_url), + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()["x-api-key"], "api-secret"); + } + + #[test] + fn identity_is_enforced_after_header_merges_for_every_request_target() { + let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); + req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); + let input_headers = HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]); + let session_headers = HashMap::from([("Accept-Encoding".to_string(), "zstd".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::Identity, + &stream_url, + &req_headers, + Some(&input_headers), + Some(&session_headers), + ); + let client = reqwest::Client::new(); + + let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); + let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); + for target in [None, Some(&same_origin), Some(&cross_origin)] { + let credential_state = if target == Some(&cross_origin) { + ProviderRequestCredentialState::Scrubbed + } else { + ProviderRequestCredentialState::OriginalOrigin + }; + let request = prepare_client(&client, &options, target, credential_state) + .0 + .build() + .unwrap(); + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-"); + if target == Some(&cross_origin) { + assert!(!request.headers().contains_key(reqwest::header::AUTHORIZATION)); + assert!(!request.headers().contains_key(reqwest::header::COOKIE)); + } + } + + let reconnect_options = options.clone(); + assert_eq!(reconnect_options.content_representation(), ProviderContentRepresentationMode::Identity); + assert_eq!(reconnect_options.hls_content_coding_object_kind(), Some(HlsOriginContentCodingObjectKind::Other)); + assert_eq!( + options.clone().for_deferred_open().hls_content_coding_object_kind(), + Some(HlsOriginContentCodingObjectKind::Other) + ); + } + + #[test] + fn preserve_origin_does_not_override_accept_encoding() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip, br".parse().unwrap()); + let options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); + + let request = + prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip, br"); + } + + #[tokio::test] + async fn automatic_redirect_round_trip_keeps_identity_and_never_restores_credentials() { + let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; + let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = + spawn_redirect_round_trip(encoded).await; + let options = redirect_test_options(&entry_url); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .expect("automatic redirect client"); + + let response = prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .send() + .await + .expect("automatic redirect request succeeds"); + + assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; + } + + #[tokio::test] + async fn manual_redirect_round_trip_keeps_identity_and_never_restores_credentials() { + let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; + let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = + spawn_redirect_round_trip(encoded).await; + let options = redirect_test_options(&entry_url); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("manual redirect client"); + + let response = send_with_manual_redirects(&client, &options, &test_app_config()) + .await + .expect("manual redirect request succeeds"); + + assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; + } + + #[tokio::test] + async fn deferred_open_without_origin_head_accepts_only_plain_origin_200() { + const IDENTITY_BODY: &[u8] = b"deferred identity body"; + + let stream_url = Url::parse("http://provider.example/live/deferred.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + let deferred_options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None) + .for_deferred_open(); + let deferred_request = prepare_client( + &reqwest::Client::new(), + &deferred_options, + None, + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .expect("deferred provider request"); + assert_eq!(deferred_options.content_representation(), ProviderContentRepresentationMode::Identity); + assert!(!deferred_options.response_head_is_available()); + assert_eq!(deferred_options.hls_content_coding_object_kind(), None); + assert_eq!(deferred_request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); + + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], encoded).await; + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await + .expect("plain 200 is safe for deferred identity streaming"); + assert_eq!(collect_stream(stream).await.expect("deferred body decodes"), IDENTITY_BODY); + let (headers, status, _, _) = info.expect("normalized deferred response metadata"); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); + + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = local_response( + StatusCode::OK, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-21/22")], + encoded, + ) + .await; + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { + status: StatusCode::OK, + has_content_range: true, + }) + )); + + let (response, _) = local_response(StatusCode::PARTIAL_CONTENT, &[], IDENTITY_BODY.to_vec()).await; + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { + status: StatusCode::PARTIAL_CONTENT, + has_content_range: false, + }) + )); + } + + #[tokio::test] + async fn deferred_open_rejects_preserve_origin_without_origin_head() { + let (response, _) = local_response(StatusCode::OK, &[], b"opaque bytes".to_vec()).await; + + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { + status: StatusCode::OK, + has_content_range: false, + }) + )); + } + + #[tokio::test] + async fn preserve_origin_rejects_invalid_representation_header_before_stream_release() { + let (response, requests) = + local_response(StatusCode::OK, &[("Content-Encoding", "gzip,,br")], b"body must not be released".to_vec()) + .await; + + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await, + Err(ProviderStreamPreparationError::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding)) + )); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn legacy_hls_segment_key_map_part_and_other_stream_as_identity() { + const IDENTITY: &[u8] = b"identity hls resource bytes"; + let cases = [ + ("segment", "gzip", TestEncoding::Gzip), + ("segment-zlib", "deflate", TestEncoding::Zlib), + ("segment-raw-deflate", "deflate", TestEncoding::RawDeflate), + ("key", "br", TestEncoding::Brotli), + ("map", "zstd", TestEncoding::Zstd), + ("part", "gzip", TestEncoding::Gzip), + ("other", "br", TestEncoding::Brotli), + ]; + + for (resource_kind, content_encoding, encoding) in cases { + let encoded = encode(IDENTITY, encoding).await; + let (response, _) = + local_response(StatusCode::OK, &[("Content-Encoding", content_encoding)], encoded).await; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), IDENTITY, "failed for {resource_kind}"); + let (headers, status, _, _) = info.unwrap(); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); + } + } + + #[tokio::test] + async fn legacy_hls_declared_only_preserves_headerless_gzip_magic() { + let body = Bytes::from_static(b"\x1f\x8bnot-an-encoded-key"); + let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; + + let (stream, _) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), body); + } + + #[tokio::test] + async fn legacy_hls_identity_keeps_unencoded_partial_content_headers() { + const PARTIAL_BODY: &[u8] = b"abc"; + let (response, _) = + local_response(StatusCode::PARTIAL_CONTENT, &[("Content-Range", "bytes 0-2/10")], PARTIAL_BODY.to_vec()) + .await; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("unencoded partial identity response is safe"); + let (headers, status, _, _) = info.expect("partial response metadata"); + + assert_eq!(collect_stream(stream).await.expect("partial body streams"), PARTIAL_BODY); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == "3")); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-2/10")); + assert!(headers.iter().all(|(name, _)| name != "content-encoding")); + } + + #[tokio::test] + async fn legacy_hls_identity_decoding_removes_stale_representation_headers() { + const IDENTITY_BODY: &[u8] = b"decoded identity response"; + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = + local_response(StatusCode::OK, &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], encoded) + .await; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("encoded full response decodes"); + let (headers, status, _, _) = info.expect("decoded response metadata"); + + assert_eq!(collect_stream(stream).await.expect("decoded body streams"), IDENTITY_BODY); + assert_eq!(status, StatusCode::OK); + for name in ["content-encoding", "content-length", "content-range"] { + assert!(headers.iter().all(|(key, _)| key != name), "stale header {name}"); + } + } + + #[tokio::test] + async fn legacy_hls_rejects_encoded_partial_content_before_client_streaming() { + let encoded = encode(b"partial identity bytes", TestEncoding::Gzip).await; + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], + encoded, + ) + .await; + + let result = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await; + + assert!(matches!( + result, + Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::EncodedPartialContent)) + )); + } + + #[tokio::test] + async fn legacy_hls_decoder_setup_obeys_provider_body_idle_timeout() { + let (response, origin_task) = local_stalled_deflate_response().await; + + let result = prepare_provider_stream_response_with_idle_timeout( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + Duration::from_millis(25), + ) + .await; + origin_task.abort(); + + assert!(matches!( + result, + Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::PrefixRead(error))) + if error.kind() == io::ErrorKind::TimedOut + )); + } + + #[tokio::test] + async fn preserve_origin_keeps_encoded_body_and_representation_headers() { + let encoded = encode(b"preserved representation", TestEncoding::Gzip).await; + let expected = encoded.clone(); + let expected_len = encoded.len().to_string(); + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 10-20/100")], + encoded, + ) + .await; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + let (headers, status, _, _) = info.unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), expected); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "gzip")); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 10-20/100")); + assert!(headers.iter().all(|(name, _)| name != "transfer-encoding")); + } + + #[tokio::test] + async fn preserve_origin_keeps_unknown_content_coding_and_body_unchanged() { + let body = b"opaque provider representation".to_vec(); + let expected = body.clone(); + let expected_len = body.len().to_string(); + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "x-provider-coding"), ("Content-Range", "bytes 0-29/100")], + body, + ) + .await; + + let (stream, info) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("valid unknown coding is preserved"); + let (headers, status, _, _) = info.expect("preserved response metadata"); + + assert_eq!(collect_stream(stream).await.expect("opaque body streams"), expected); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "x-provider-coding")); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-29/100")); + } + + #[tokio::test] + async fn preserve_origin_never_magic_sniffs_headerless_body() { + let body = Bytes::from_static(b"\x1f\x8bopaque non-hls bytes"); + let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; + + let (stream, _) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), body); + } + + #[tokio::test] + async fn legacy_hls_stream_decoder_failure_aborts_without_retry() { + let mut truncated = encode(b"decoder failure after response headers", TestEncoding::Gzip).await; + truncated.truncate(truncated.len().saturating_sub(5)); + let (response, requests) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], truncated).await; + + let (stream, _) = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + let error = collect_stream(stream).await.expect_err("truncated gzip must terminate the body stream"); + + assert!(matches!(error, StreamError::ContentDecoding(_))); + assert_eq!(requests.load(Ordering::SeqCst), 1, "a body-stream failure cannot trigger a new request"); + } } diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index 4fd94f420..c8977c366 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -4074,6 +4074,7 @@ pub fn create_panel_api_provisioning_stream_details( disable_provider_grace: true, reconnect_flag: None, provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, }; } @@ -4100,6 +4101,7 @@ pub fn create_panel_api_provisioning_stream_details( disable_provider_grace: true, reconnect_flag: None, provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, grace_resolution_context: None, } } diff --git a/backend/src/processing/parser/hls/transient_manifest.rs b/backend/src/processing/parser/hls/transient_manifest.rs index f5516fd08..39fd1c06f 100644 --- a/backend/src/processing/parser/hls/transient_manifest.rs +++ b/backend/src/processing/parser/hls/transient_manifest.rs @@ -1,11 +1,12 @@ use super::rewrite_hls_url; -use crate::api::model::{ - ProxySessionId, TransientResourceId, TransientResourceKind, TransientResourceRef, - HLS_ACCESS_LEASE_ID_PLACEHOLDER, +use crate::{ + api::model::{ + ProxySessionId, TransientResourceId, TransientResourceKind, TransientResourceRef, + HLS_ACCESS_LEASE_ID_PLACEHOLDER, + }, + model::StripConfig, }; -use crate::model::StripConfig; -use shared::model::HlsStripMode; -use shared::utils::CONSTANTS; +use shared::{model::HlsStripMode, utils::CONSTANTS}; use std::{collections::HashMap, time::Duration}; use url::Url; @@ -83,20 +84,14 @@ impl TransientManifestRewriter { } if body.is_empty() { - return TransientRewriteResult { - body: rewritten_body, - resources: Vec::new(), - }; + return TransientRewriteResult { body: rewritten_body, resources: Vec::new() }; } if let Some(discontinuity_sequence) = options.handoff_discontinuity_sequence { rewritten_body = apply_handoff_discontinuity_boundary(&rewritten_body, discontinuity_sequence); } - TransientRewriteResult { - body: rewritten_body, - resources: resources.into_values().collect(), - } + TransientRewriteResult { body: rewritten_body, resources: resources.into_values().collect() } } } @@ -142,12 +137,8 @@ pub fn materialize_transient_provisioning_handoff_view( &previous_units, previous_units.len().saturating_sub(selected_previous_units.len()), ); - let mut rewritten = String::with_capacity( - origin_body - .len() - .saturating_add(previous_provisioning_body.len()) - .saturating_add(64), - ); + let mut rewritten = + String::with_capacity(origin_body.len().saturating_add(previous_provisioning_body.len()).saturating_add(64)); for line in &origin_lines[..first_origin_unit.start] { if line.line.trim().starts_with("#EXT-X-MEDIA-SEQUENCE:") { if let Some(media_sequence) = media_sequence { @@ -213,19 +204,12 @@ pub fn apply_transient_discontinuity_sequence(body: &str, discontinuity_sequence } pub fn transient_discontinuity_sequence(body: &str) -> Option { - manifest_lines(body) - .iter() - .find_map(|line| parse_discontinuity_sequence(line.line)) + manifest_lines(body).iter().find_map(|line| parse_discontinuity_sequence(line.line)) } pub fn transient_visible_discontinuity_count(body: &str) -> u64 { - u64::try_from( - manifest_lines(body) - .iter() - .filter(|line| is_discontinuity_tag(line.line.trim())) - .count(), - ) - .unwrap_or(u64::MAX) + u64::try_from(manifest_lines(body).iter().filter(|line| is_discontinuity_tag(line.line.trim())).count()) + .unwrap_or(u64::MAX) } fn rewrite_line( @@ -268,6 +252,21 @@ fn rewrite_line( ); } + if trimmed.starts_with("#EXT-X-PART:") + || (trimmed.starts_with("#EXT-X-PRELOAD-HINT:") && trimmed.contains("TYPE=PART")) + { + return rewrite_uri_attribute( + line, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + TransientResourceKind::Part, + ); + } + if trimmed.starts_with('#') && CONSTANTS.re_hls_uri.is_match(line) { return rewrite_uri_attribute( line, @@ -323,10 +322,7 @@ fn rewrite_uri_attribute( resources, kind, ); - CONSTANTS - .re_hls_uri - .replace(line, format!(r#"URI="{proxy_uri}""#)) - .to_string() + CONSTANTS.re_hls_uri.replace(line, format!(r#"URI="{proxy_uri}""#)).to_string() } #[allow(clippy::too_many_arguments)] @@ -353,10 +349,7 @@ fn rewrite_resource_uri( Some(extension.clone()), ); let resource_id = resource.id.clone(); - resources - .entry(resource_id.clone()) - .and_modify(|existing| *existing = resource.clone()) - .or_insert(resource); + resources.entry(resource_id.clone()).and_modify(|existing| *existing = resource.clone()).or_insert(resource); ( format!( "/hls/shared/live/{}/{}/r/{}.{}", @@ -381,12 +374,8 @@ fn extract_extension(resolved_origin_uri: &str) -> Option { .filter(|extension| !extension.is_empty()); } - let without_query = resolved_origin_uri - .split_once('?') - .map_or(resolved_origin_uri, |(path, _)| path); - let without_fragment = without_query - .split_once('#') - .map_or(without_query, |(path, _)| path); + let without_query = resolved_origin_uri.split_once('?').map_or(resolved_origin_uri, |(path, _)| path); + let without_fragment = without_query.split_once('#').map_or(without_query, |(path, _)| path); without_fragment .rsplit_once('.') .map(|(_, extension)| extension.to_string()) @@ -397,7 +386,7 @@ fn fallback_extension(kind: TransientResourceKind) -> &'static str { match kind { TransientResourceKind::Key => "key", TransientResourceKind::Map => "mp4", - TransientResourceKind::Segment | TransientResourceKind::Other => "bin", + TransientResourceKind::Segment | TransientResourceKind::Part | TransientResourceKind::Other => "bin", } } @@ -516,12 +505,7 @@ fn append_media_unit_with_duration_override( } } -fn append_gap_media_unit( - output: &mut String, - lines: &[ManifestLine<'_>], - unit: MediaSegmentUnit, - duration_ms: u64, -) { +fn append_gap_media_unit(output: &mut String, lines: &[ManifestLine<'_>], unit: MediaSegmentUnit, duration_ms: u64) { output.push_str("#EXT-X-GAP\n"); let uri_index = lines[unit.start..=unit.end] .iter() @@ -551,9 +535,7 @@ fn append_manifest_block_separator(output: &mut String) { } fn media_unit_has_discontinuity(lines: &[ManifestLine<'_>], unit: MediaSegmentUnit) -> bool { - lines[unit.start..=unit.end] - .iter() - .any(|line| is_discontinuity_tag(line.line.trim())) + lines[unit.start..=unit.end].iter().any(|line| is_discontinuity_tag(line.line.trim())) } fn handoff_provisioning_media_sequence( @@ -576,12 +558,9 @@ fn format_duration_ms(duration_ms: u64) -> String { format!("{}.{:03}", duration fn apply_handoff_discontinuity_boundary(body: &str, handoff_discontinuity_sequence: u64) -> String { let lines = manifest_lines(body); - let origin_discontinuity_sequence = lines - .iter() - .find_map(|line| parse_discontinuity_sequence(line.line)) - .unwrap_or(0); - let effective_discontinuity_sequence = - origin_discontinuity_sequence.saturating_add(handoff_discontinuity_sequence); + let origin_discontinuity_sequence = + lines.iter().find_map(|line| parse_discontinuity_sequence(line.line)).unwrap_or(0); + let effective_discontinuity_sequence = origin_discontinuity_sequence.saturating_add(handoff_discontinuity_sequence); let existing_sequence_index = lines.iter().position(|line| is_discontinuity_sequence_tag(line.line.trim())); let insert_sequence_index = existing_sequence_index.unwrap_or_else(|| discontinuity_sequence_insert_index(&lines)); let first_segment = first_media_segment_boundary(&lines); @@ -656,9 +635,7 @@ fn parse_discontinuity_sequence(line: &str) -> Option { line.trim().strip_prefix("#EXT-X-DISCONTINUITY-SEQUENCE:")?.trim().parse().ok() } -fn is_discontinuity_sequence_tag(line: &str) -> bool { - line.starts_with("#EXT-X-DISCONTINUITY-SEQUENCE:") -} +fn is_discontinuity_sequence_tag(line: &str) -> bool { line.starts_with("#EXT-X-DISCONTINUITY-SEQUENCE:") } fn is_discontinuity_tag(line: &str) -> bool { line == "#EXT-X-DISCONTINUITY" } @@ -670,9 +647,7 @@ fn duration_ms_from_extinf(value: &str) -> Option { u64::try_from(Duration::from_secs_f64(seconds).as_millis()).ok() } -fn is_segment_unit_tag(line: &str) -> bool { - is_media_segment_unit_tag(line) -} +fn is_segment_unit_tag(line: &str) -> bool { is_media_segment_unit_tag(line) } fn is_media_segment_unit_tag(line: &str) -> bool { line.starts_with("#EXTINF:") @@ -694,14 +669,23 @@ mod tests { apply_transient_discontinuity_sequence, materialize_transient_provisioning_handoff_view, transient_discontinuity_sequence, TransientManifestRewriter, TransientRewriteOptions, }; - use crate::api::model::{build_transient_resource_id, ProxySessionId, TransientResourceKind}; - use crate::model::StripConfig; -use shared::model::HlsStripMode; + use crate::{ + api::model::{build_transient_resource_id, ProxySessionId, TransientResourceKind}, + model::StripConfig, + }; + use shared::model::HlsStripMode; const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; fn rewrite(body: &str) -> crate::processing::parser::hls::transient_manifest::TransientRewriteResult { - TransientManifestRewriter::rewrite(body, BASE_URL, &ProxySessionId("proxy-id".to_string()), b"secret", 100, 1_000) + TransientManifestRewriter::rewrite( + body, + BASE_URL, + &ProxySessionId("proxy-id".to_string()), + b"secret", + 100, + 1_000, + ) } fn rewrite_with_handoff(body: &str) -> crate::processing::parser::hls::transient_manifest::TransientRewriteResult { @@ -712,9 +696,7 @@ use shared::model::HlsStripMode; b"secret", 100, 1_000, - TransientRewriteOptions { - handoff_discontinuity_sequence: Some(0), - }, + TransientRewriteOptions { handoff_discontinuity_sequence: Some(0) }, ) } @@ -722,7 +704,9 @@ use shared::model::HlsStripMode; fn ext_x_key_uri_is_rewritten_without_changing_other_attributes() { let result = rewrite("#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"keys/key.bin\",IV=0x1\n#EXTINF:4.0,\nseg.ts\n"); - assert!(result.body.contains("#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result + .body + .contains("#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); assert!(result.body.contains(".bin\",IV=0x1")); assert!(!result.body.contains("keys/key.bin")); } @@ -767,12 +751,16 @@ use shared::model::HlsStripMode; } #[test] - fn unsupported_uri_attributes_are_rewritten_to_transient_resources() { + fn low_latency_part_uri_is_rewritten_with_part_diagnostics_kind() { let result = rewrite("#EXTM3U\n#EXT-X-PART:DURATION=1.0,URI=\"parts/part001.m4s\"\n"); - assert!(result.body.contains("#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result + .body + .contains("#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); assert!(result.body.contains(".m4s\"")); assert!(!result.body.contains("parts/part001.m4s")); + assert_eq!(result.resources.len(), 1); + assert_eq!(result.resources[0].kind, TransientResourceKind::Part); } #[test] @@ -888,7 +876,9 @@ use shared::model::HlsStripMode; assert!(body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:0\n\n#EXTINF:2.000,")); assert_eq!(body.matches("#EXTINF:").count(), 6); assert!(body.contains("#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000002.ts?pseq=2")); - assert!(body.contains("\n#EXT-X-GAP\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000003.ts?pseq=3")); + assert!(body.contains( + "\n#EXT-X-GAP\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000003.ts?pseq=3" + )); assert!(body.contains("\n#EXT-X-DISCONTINUITY\n#EXTINF:1.92,")); let gap = body.find("\n#EXT-X-GAP\n").expect("gap tag"); let discontinuity = body.find("\n#EXT-X-DISCONTINUITY\n#EXTINF:1.92,").expect("handoff discontinuity"); @@ -907,5 +897,4 @@ use shared::model::HlsStripMode; } fn strip_segments(value: u64) -> StripConfig { StripConfig { mode: HlsStripMode::Segments, value } } - } diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 99eeba375..973601280 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -428,6 +428,7 @@ fn process_header_internal( } } + plih.freeze_input_stream_id(); plih } diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 3f50676db..c5f3f3f01 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -105,7 +105,7 @@ pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStre } - PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { uuid: generate_provider_playlist_uuid(&input.name, &episode_id, PlaylistItemType::Series), id: episode_id.into(), @@ -124,8 +124,10 @@ pub fn parse_xtream_series_info(parent_uuid: &UUIDType, series_info: &SeriesStre // Keep episode ordering tied to its parent SeriesInfo to avoid cross-series ordinal overlap. source_ordinal: parent_source_ordinal, ..Default::default() - } - } + }, + }; + item.header.freeze_input_stream_id(); + item }).collect(); return if result.is_empty() { None } else { Some(result) }; } @@ -199,7 +201,7 @@ pub async fn parse_xtream(input: &ConfigInput, let category_name = &group.category_name; let stream_url = create_xtream_url(xtream_cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); let item_type = PlaylistItemType::from(xtream_cluster); - let item = PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { id: stream.get_stream_id().intern(), uuid: generate_provider_playlist_uuid(&input_name, &stream.get_stream_id().to_string(), item_type), @@ -217,6 +219,7 @@ pub async fn parse_xtream(input: &ConfigInput, ..Default::default() }, }; + item.header.freeze_input_stream_id(); group.add(item); } @@ -371,7 +374,7 @@ where let stream_url = create_xtream_url(cluster, url, username, password, &stream, live_stream_use_prefix, live_stream_without_extension); let item_type = PlaylistItemType::from(cluster); - let item = PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { id: stream.get_stream_id().intern(), uuid: generate_provider_playlist_uuid(input_name, &stream.get_stream_id().to_string(), item_type), @@ -390,6 +393,7 @@ where ..Default::default() }, }; + item.header.freeze_input_stream_id(); // if let Some(StreamProperties::Series(props)) = item.header.additional_properties.as_mut() { // // We need to set category_id for Series properties just like parse_xtream might expect or use? diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index dd4ec219c..f10cd698c 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -1044,6 +1044,13 @@ fn execute_pipe<'a>( }; let mut new_fpl = FetchedPlaylist { input: fpl.input, source, epg: fpl.epg.clone() }; + // In-memory items are frozen here at the target-processing boundary. Read-only disk sources + // capture the same identity when their persisted M3U/Xtream items are converted to PlaylistItem. + if new_fpl.is_memory() { + for item in new_fpl.items_mut() { + item.header.freeze_input_stream_id(); + } + } if target.options.as_ref().is_some_and(|opt| opt.remove_duplicates) { new_fpl.deduplicate(duplicates); } @@ -1664,15 +1671,194 @@ pub async fn exec_processing( #[cfg(test)] mod tests { use super::*; - use shared::foundation::{get_filter, ValueProvider}; - use shared::model::{ClusterFlags, PlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster}; + use shared::foundation::{get_filter, MapperScript, ValueProvider}; + use shared::model::{ + ClusterFlags, ConfigRenameDto, ConfigTargetDto, M3uPlaylistItem, PlaylistEntry, PlaylistItem, + PlaylistItemHeader, PlaylistItemType, XtreamCluster, XtreamPlaylistItem, + }; use shared::utils::Internable; + fn serialize_without_trailing_input_stream_id(value: &T) -> Vec { + let mut encoded = rmp_serde::to_vec(value).expect("playlist item should serialize"); + assert_eq!(encoded.pop(), Some(0xa0), "input_stream_id should be the trailing empty string"); + match encoded[0] { + marker @ 0x91..=0x9f => encoded[0] = marker - 1, + 0xdc => { + let len = u16::from_be_bytes([encoded[1], encoded[2]]); + encoded[1..3].copy_from_slice(&(len - 1).to_be_bytes()); + } + 0xdd => { + let len = u32::from_be_bytes([encoded[1], encoded[2], encoded[3], encoded[4]]); + encoded[1..5].copy_from_slice(&(len - 1).to_be_bytes()); + } + marker => panic!("unexpected MessagePack sequence marker {marker:#x}"), + } + encoded + } + fn item_with_props(props: StreamProperties) -> PlaylistItem { let header = shared::model::PlaylistItemHeader { additional_properties: Some(props), ..Default::default() }; PlaylistItem { header } } + #[test] + fn rename_preserves_input_stream_id_captured_at_target_boundary() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "origin-alpha".intern(), + url: "http://provider.example/channel.m3u8".intern(), + ..Default::default() + }, + }; + item.header.freeze_input_stream_id(); + let rename = ConfigRename::from(&ConfigRenameDto { + field: ItemField::Url, + pattern: "provider".to_string(), + new_name: "target".to_string(), + t_pattern: None, + }); + + exec_rename(&mut item, Some(&vec![rename])); + + assert_eq!(item.header.url.as_ref(), "http://target.example/channel.m3u8"); + assert_eq!(item.header.input_stream_id.as_ref(), "origin-alpha"); + } + + #[test] + fn mapper_changes_id_without_changing_frozen_input_stream_id() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "origin-alpha".intern(), + name: "Channel".intern(), + ..Default::default() + }, + }; + item.header.freeze_input_stream_id(); + let mapping = Mapping { + mapper: Some(vec![crate::model::Mapper { + filter: r#"name ~ ".*""#.to_string(), + script: r#"@id = "target-id""#.to_string(), + t_filter: Some(get_filter(r#"name ~ ".*""#, None).expect("filter should parse")), + t_script: Some(MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse")), + }]), + ..Default::default() + }; + + let (mapped, _, matched) = map_channel(item, &mapping); + + assert!(matched); + assert_eq!(mapped.header.id.as_ref(), "target-id"); + assert_eq!(mapped.header.input_stream_id.as_ref(), "origin-alpha"); + } + + #[test] + fn mapper_cannot_resurrect_missing_legacy_input_stream_id_from_target_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + source.header.freeze_input_stream_id(); + let mut legacy_xtream = XtreamPlaylistItem::from(&source); + legacy_xtream.provider_id = 0; + legacy_xtream.input_stream_id = "".intern(); + legacy_xtream.url = "http://provider.example/live/channel.m3u8".intern(); + let mut legacy_item = PlaylistItem::from(&legacy_xtream); + legacy_item.header.freeze_input_stream_id(); + let mapping = Mapping { + mapper: Some(vec![crate::model::Mapper { + filter: r#"name ~ ".*""#.to_string(), + script: r#"@id = "target-id""#.to_string(), + t_filter: Some(get_filter(r#"name ~ ".*""#, None).expect("filter should parse")), + t_script: Some(MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse")), + }]), + ..Default::default() + }; + + let (mapped, _, matched) = map_channel(legacy_item, &mapping); + let materialized_m3u = M3uPlaylistItem::from(&mapped); + let materialized_xtream = XtreamPlaylistItem::from(&mapped); + + assert!(matched); + assert_eq!(mapped.header.id.as_ref(), "target-id"); + assert_eq!(mapped.get_input_stream_id(), None); + assert!(materialized_m3u.provider_id.is_empty()); + assert_eq!(materialized_m3u.get_input_stream_id(), None); + assert_eq!(materialized_xtream.provider_id, 0); + assert_eq!(materialized_xtream.get_input_stream_id(), None); + } + + #[test] + fn execute_pipe_freezes_input_stream_id_without_rename_or_mapper() { + let input = ConfigInput::default(); + let item = PlaylistItem { + header: PlaylistItemHeader { id: "origin-alpha".intern(), ..Default::default() }, + }; + let source = MemoryPlaylistSource::new(vec![PlaylistGroup { + id: 1, + title: "Group".intern(), + channels: vec![item], + xtream_cluster: XtreamCluster::Live, + }]) + .into_source(); + let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; + let mut duplicates = HashSet::new(); + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let mut processed = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false) + .expect("target processing should succeed"); + let mut groups = processed.source.take_groups(); + + assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); + assert!(groups[0].channels[0].header.set_field("id", "late-target-id")); + assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); + } + + #[test] + fn legacy_messagepack_playlist_items_default_missing_input_stream_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "origin-alpha".intern(), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + + let header_bytes = serialize_without_trailing_input_stream_id(&source.header); + let decoded_header: PlaylistItemHeader = + rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize"); + assert!(decoded_header.input_stream_id.is_empty()); + assert_eq!(decoded_header.get_input_stream_id(), None); + let mut decoded_header = decoded_header; + decoded_header.freeze_input_stream_id(); + assert_eq!(decoded_header.get_input_stream_id().as_deref(), Some("origin-alpha")); + + source.header.freeze_input_stream_id(); + let mut m3u_item = M3uPlaylistItem::from(&source); + m3u_item.input_stream_id = "".intern(); + let m3u_bytes = serialize_without_trailing_input_stream_id(&m3u_item); + let decoded_m3u: M3uPlaylistItem = + rmp_serde::from_slice(&m3u_bytes).expect("legacy M3U item should deserialize"); + assert!(decoded_m3u.input_stream_id.is_empty()); + assert_eq!(decoded_m3u.get_input_stream_id().as_deref(), Some("origin-alpha")); + + let mut xtream_item = XtreamPlaylistItem::from(&source); + xtream_item.input_stream_id = "".intern(); + let xtream_bytes = serialize_without_trailing_input_stream_id(&xtream_item); + let decoded_xtream: XtreamPlaylistItem = + rmp_serde::from_slice(&xtream_bytes).expect("legacy Xtream item should deserialize"); + assert!(decoded_xtream.input_stream_id.is_empty()); + assert_eq!(decoded_xtream.get_input_stream_id().as_deref(), Some("80510")); + } + #[test] fn has_probe_details_requires_video_and_audio_for_video() { let video = shared::model::VideoStreamProperties { diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs index eb73e1e3c..9895c265b 100644 --- a/backend/src/repository/library_repository.rs +++ b/backend/src/repository/library_repository.rs @@ -192,6 +192,7 @@ mod tests { input_name: "lib".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "1".intern(), } } @@ -233,6 +234,7 @@ mod tests { input_name: "lib".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "2".intern(), } } diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index f5fb9b5b1..a5aa1dff2 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -440,6 +440,7 @@ mod tests { t_catchup_mode: None, source_ordinal: 0, additional_properties: None, + input_stream_id: "813294".intern(), } } diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 9c112de33..1cac3dace 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -1158,6 +1158,7 @@ mod tests { input_name: Arc::clone(&input_name), channel_no: 0, source_ordinal: 0, + input_stream_id: "9001".intern(), }; let provider_parent_code = xtream_series_info.get_uuid().intern(); let xtream_provider_episode = XtreamPlaylistItem { @@ -1179,6 +1180,7 @@ mod tests { input_name, channel_no: 0, source_ordinal: 0, + input_stream_id: "201".intern(), }; let provider_episode = PlaylistItem::from(&xtream_provider_episode); let mut series_info = PlaylistItem::from(&xtream_series_info); diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 7bf1ceace..650c20dec 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1716,6 +1716,7 @@ mod tests { input_name: "input_a".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: provider_id.to_string().intern(), } } diff --git a/backend/src/utils/compression/compressed_file_reader.rs b/backend/src/utils/compression/compressed_file_reader.rs index 33bc34e22..3eb46ed09 100644 --- a/backend/src/utils/compression/compressed_file_reader.rs +++ b/backend/src/utils/compression/compressed_file_reader.rs @@ -1,8 +1,12 @@ -use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; -use std::path::Path; +use crate::utils::{ + compression::compression_utils::{is_gzip, is_zlib_header}, + file_reader, open_readonly_file, +}; use flate2::bufread::{GzDecoder, ZlibDecoder}; -use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; -use crate::utils::{file_reader, open_readonly_file}; +use std::{ + io::{BufRead, BufReader, Read, Seek, SeekFrom}, + path::Path, +}; pub struct CompressedFileReader { reader: BufReader>, @@ -19,38 +23,29 @@ impl CompressedFileReader { let reader: Box = if is_gzip(&header) { Box::new(GzDecoder::new(buffered_file)) - } else if is_deflate(&header) { + } else if is_zlib_header(&header) { Box::new(ZlibDecoder::new(buffered_file)) } else { Box::new(buffered_file) }; - Ok(Self { - reader: file_reader(reader), - }) + Ok(Self { reader: file_reader(reader) }) } } // Implement the Read trait for CompressedFileReader impl Read for CompressedFileReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.reader.read(buf) - } + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.reader.read(buf) } } // Implement BufRead for CompressedFileReader impl BufRead for CompressedFileReader { - fn fill_buf(&mut self) -> std::io::Result<&[u8]> { - self.reader.fill_buf() - } + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { self.reader.fill_buf() } - fn consume(&mut self, amt: usize) { - self.reader.consume(amt); - } + fn consume(&mut self, amt: usize) { self.reader.consume(amt); } } -impl Iterator for CompressedFileReader -{ +impl Iterator for CompressedFileReader { type Item = std::io::Result; fn next(&mut self) -> Option { diff --git a/backend/src/utils/compression/compressed_file_reader_async.rs b/backend/src/utils/compression/compressed_file_reader_async.rs index a62c92b45..302a03a3a 100644 --- a/backend/src/utils/compression/compressed_file_reader_async.rs +++ b/backend/src/utils/compression/compressed_file_reader_async.rs @@ -1,12 +1,16 @@ -use crate::utils::async_file_reader; -use crate::utils::compression::compression_utils::{is_deflate, is_gzip}; +use crate::utils::{ + async_file_reader, + compression::compression_utils::{is_gzip, is_zlib_header}, +}; use async_compression::tokio::bufread::{GzipDecoder, ZlibDecoder}; -use std::path::Path; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::fs::File; -use tokio::io::{ - self, AsyncRead, AsyncReadExt, AsyncSeekExt, ReadBuf, +use std::{ + path::Path, + pin::Pin, + task::{Context, Poll}, +}; +use tokio::{ + fs::File, + io::{self, AsyncRead, AsyncReadExt, AsyncSeekExt, ReadBuf}, }; pub struct CompressedFileReaderAsync { @@ -24,7 +28,7 @@ impl CompressedFileReaderAsync { if is_gzip(&header) { Ok(Self { reader: Box::new(GzipDecoder::new(buffered_file)) }) - } else if is_deflate(&header) { + } else if is_zlib_header(&header) { Ok(Self { reader: Box::new(ZlibDecoder::new(buffered_file)) }) } else { Ok(Self { reader: Box::new(buffered_file) }) @@ -33,12 +37,7 @@ impl CompressedFileReaderAsync { } impl AsyncRead for CompressedFileReaderAsync { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { Pin::new(&mut self.reader).poll_read(cx, buf) } } - diff --git a/backend/src/utils/compression/compression_utils.rs b/backend/src/utils/compression/compression_utils.rs index f59420533..bdc531a72 100644 --- a/backend/src/utils/compression/compression_utils.rs +++ b/backend/src/utils/compression/compression_utils.rs @@ -1,15 +1,21 @@ +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use std::io::{Read, Write}; -use flate2::Compression; -use flate2::read::GzDecoder; -use flate2::write::GzEncoder; pub const fn is_gzip(bytes: &[u8]) -> bool { // Gzip files start with the bytes 0x1F 0x8B bytes.len() >= 2 && bytes[0] == 0x1F && bytes[1] == 0x8B } -pub const fn is_deflate(bytes: &[u8]) -> bool { - bytes[0] == 0x78 && (bytes[1] == 0x01 || bytes[1] == 0x9C || bytes[1] == 0xDA) +/// Reports whether the prefix is a valid RFC 1950 zlib header. +pub const fn is_zlib_header(bytes: &[u8]) -> bool { + if bytes.len() < 2 { + return false; + } + + let compression_method_and_info = bytes[0]; + let flags = bytes[1]; + let header = u16::from_be_bytes([compression_method_and_info, flags]); + compression_method_and_info & 0x0f == 8 && compression_method_and_info >> 4 <= 7 && header.rem_euclid(31) == 0 } pub fn compress_string(input: &str) -> std::io::Result> { @@ -24,3 +30,25 @@ pub fn decompress_string(input: &[u8]) -> std::io::Result { decoder.read_to_string(&mut decompressed)?; Ok(decompressed) } + +#[cfg(test)] +mod tests { + use super::is_zlib_header; + + #[test] + fn zlib_header_validation_covers_method_window_checksum_and_short_prefixes() { + for valid in [[0x78, 0x01], [0x78, 0x5e], [0x78, 0x9c], [0x78, 0xda]] { + assert!(is_zlib_header(&valid), "expected valid zlib header {valid:02x?}"); + } + + for invalid in [ + &[][..], + &[0x78][..], + &[0x79, 0x18][..], // unsupported compression method + &[0x88, 0x1c][..], // CINFO exceeds the RFC 1950 maximum window size + &[0x78, 0x02][..], // invalid FCHECK remainder + ] { + assert!(!is_zlib_header(invalid), "expected invalid zlib header {invalid:02x?}"); + } + } +} diff --git a/backend/src/utils/db_viewer.rs b/backend/src/utils/db_viewer.rs index 57d9b4dd5..353c02116 100644 --- a/backend/src/utils/db_viewer.rs +++ b/backend/src/utils/db_viewer.rs @@ -335,6 +335,7 @@ mod tests { input_name: "input".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "52568".intern(), }; let value = to_human_readable_json_value(&item).expect("dump value should serialize"); diff --git a/backend/src/utils/network/content_coding.rs b/backend/src/utils/network/content_coding.rs new file mode 100644 index 000000000..394248e70 --- /dev/null +++ b/backend/src/utils/network/content_coding.rs @@ -0,0 +1,917 @@ +use super::DynReader; +use crate::utils::compression::compression_utils::{is_gzip, is_zlib_header}; +use async_compression::tokio::bufread::{BrotliDecoder, DeflateDecoder, GzipDecoder, ZlibDecoder, ZstdDecoder}; +use futures::TryStreamExt; +use reqwest::{ + header::{ + self, HeaderMap, HeaderName, HeaderValue, ACCEPT_RANGES, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_RANGE, ETAG, + TRANSFER_ENCODING, VARY, + }, + StatusCode, +}; +use std::{ + error::Error as StdError, + io, + pin::Pin, + task::{Context, Poll}, +}; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; +use tokio_util::io::StreamReader; +use url::Url; + +const MAX_MAGIC_PREFIX_LEN: usize = 4; +const ZSTD_MAGIC: [u8; 4] = [0x28, 0xb5, 0x2f, 0xfd]; + +/// Controls the final `Accept-Encoding` representation requested from an origin. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum OutboundContentCodingPolicy { + #[default] + Inherit, + Identity, +} + +/// Supported non-identity HTTP content codings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ContentCoding { + Gzip, + Deflate, + Brotli, + Zstd, +} + +impl ContentCoding { + /// Returns the fixed HTTP token used in redacted diagnostics. + pub(crate) const fn as_http_token(self) -> &'static str { + match self { + Self::Gzip => "gzip", + Self::Deflate => "deflate", + Self::Brotli => "br", + Self::Zstd => "zstd", + } + } +} + +/// Safe fixed-field projection of an origin response whose coding was removed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ContentCodingObservation { + pub(crate) content_encoding: &'static str, + pub(crate) status: StatusCode, + pub(crate) content_length: Option, +} + +/// Selects the narrowly scoped fallback detection used when `Content-Encoding` is absent. +/// +/// The shared `Declared` prefix deliberately makes header precedence explicit at every call site. +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum ContentCodingDetection { + /// Uses only declared `Content-Encoding` values. + #[default] + DeclaredOnly, + /// Preserves the historical generic-text fallback for headerless gzip and zlib only. + DeclaredOrLegacyTextMagic, + /// Detects the known headerless gzip, zlib, and Zstandard signatures accepted for HLS manifests. + DeclaredOrKnownHlsManifestMagic, +} + +/// Origin response normalized to an asynchronous identity-representation body. +pub struct DecodedHttpResponse { + pub(crate) status: StatusCode, + pub(crate) final_url: Url, + pub(crate) headers: HeaderMap, + pub(crate) body: DynReader, + decoded_from: Vec, + original_content_length: Option, +} + +impl DecodedHttpResponse { + /// Reports whether at least one non-identity content coding was removed. + pub(crate) fn was_content_decoded(&self) -> bool { !self.decoded_from.is_empty() } + + /// Returns only fixed and numeric fields suitable for HLS origin diagnostics. + pub(crate) fn content_coding_observation(&self) -> Option { + let content_encoding = match self.decoded_from.as_slice() { + [] => return None, + [coding] => coding.as_http_token(), + [_, ..] => "multiple", + }; + Some(ContentCodingObservation { + content_encoding, + status: self.status, + content_length: self.original_content_length, + }) + } +} + +/// Errors raised while parsing or preparing an HTTP content-coding chain. +#[derive(Debug, thiserror::Error)] +pub(crate) enum ContentCodingError { + #[error("invalid Content-Encoding header")] + InvalidHeader, + + #[error("unsupported Content-Encoding")] + Unsupported(String), + + #[error("encoded partial content cannot be decoded safely")] + EncodedPartialContent, + + #[error("failed to inspect encoded response prefix")] + PrefixRead(#[from] io::Error), +} + +/// Reports that a declared `Content-Encoding` field is not a valid HTTP token list. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("invalid Content-Encoding header")] +pub(crate) struct InvalidContentEncodingHeader; + +/// Typed source attached to I/O failures emitted by a streaming decoder. +#[derive(Debug, thiserror::Error)] +#[error("content decoding failed: coding={coding:?}")] +pub(crate) struct ContentDecodingIoError { + pub(crate) coding: ContentCoding, +} + +/// Errors raised while fully consuming a decoded, size-bounded text body. +#[derive(Debug, thiserror::Error)] +pub(crate) enum ContentBodyReadError { + #[error("decoded body exceeds configured limit {limit}")] + LimitExceeded { limit: usize }, + + #[error("decoded body is not valid UTF-8: valid_up_to={valid_up_to} error_len={error_len:?}")] + InvalidUtf8 { valid_up_to: usize, error_len: Option }, + + #[error(transparent)] + Io(#[from] io::Error), +} + +pub(crate) fn force_accept_encoding_identity(headers: &mut HeaderMap) { + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("identity")); +} + +pub(crate) fn apply_outbound_content_coding_policy(headers: &mut HeaderMap, policy: OutboundContentCodingPolicy) { + if matches!(policy, OutboundContentCodingPolicy::Identity) { + force_accept_encoding_identity(headers); + } +} + +/// Parses declared content-coding tokens without deciding whether Tuliprox can decode them. +pub(crate) fn parse_content_encoding_tokens(headers: &HeaderMap) -> Result, InvalidContentEncodingHeader> { + let mut tokens = Vec::new(); + for value in headers.get_all(CONTENT_ENCODING) { + let value = value.to_str().map_err(|_| InvalidContentEncodingHeader)?; + for token in value.split(',') { + let token = token.trim_matches(|character| matches!(character, ' ' | '\t')); + if !is_http_token(token) { + return Err(InvalidContentEncodingHeader); + } + tokens.push(token.to_owned()); + } + } + Ok(tokens) +} + +fn is_http_token(value: &str) -> bool { + !value.is_empty() + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' + ) + }) +} + +pub(crate) fn parse_content_codings(headers: &HeaderMap) -> Result, ContentCodingError> { + let mut codings = Vec::new(); + + for token in parse_content_encoding_tokens(headers).map_err(|_| ContentCodingError::InvalidHeader)? { + let coding = if token.eq_ignore_ascii_case("identity") { + None + } else if token.eq_ignore_ascii_case("gzip") || token.eq_ignore_ascii_case("x-gzip") { + Some(ContentCoding::Gzip) + } else if token.eq_ignore_ascii_case("deflate") { + Some(ContentCoding::Deflate) + } else if token.eq_ignore_ascii_case("br") { + Some(ContentCoding::Brotli) + } else if token.eq_ignore_ascii_case("zstd") { + Some(ContentCoding::Zstd) + } else { + return Err(ContentCodingError::Unsupported(token)); + }; + + if let Some(coding) = coding { + codings.push(coding); + } + } + + Ok(codings) +} + +pub(crate) async fn decode_response_to_identity( + response: reqwest::Response, + detection: ContentCodingDetection, +) -> Result { + let status = response.status(); + let final_url = response.url().clone(); + let headers = response.headers().clone(); + let original_content_length = + headers.get(CONTENT_LENGTH).and_then(|value| value.to_str().ok()).and_then(|value| value.parse::().ok()); + let has_content_encoding = headers.contains_key(CONTENT_ENCODING); + let mut decoded_from = parse_content_codings(&headers)?; + + let stream = response.bytes_stream().map_err(|error| { + let kind = if error.is_timeout() { io::ErrorKind::TimedOut } else { io::ErrorKind::Other }; + io::Error::new(kind, error) + }); + let mut body: DynReader = Box::pin(StreamReader::new(stream)); + + if !has_content_encoding && !matches!(detection, ContentCodingDetection::DeclaredOnly) { + let (replayed_body, prefix) = inspect_prefix(body, MAX_MAGIC_PREFIX_LEN).await?; + body = replayed_body; + let detected = match detection { + ContentCodingDetection::DeclaredOnly => None, + ContentCodingDetection::DeclaredOrLegacyTextMagic => coding_from_legacy_text_magic(&prefix), + ContentCodingDetection::DeclaredOrKnownHlsManifestMagic => coding_from_known_manifest_magic(&prefix), + }; + if let Some(coding) = detected { + decoded_from.push(coding); + } + } + + if status == StatusCode::PARTIAL_CONTENT && !decoded_from.is_empty() { + return Err(ContentCodingError::EncodedPartialContent); + } + + for coding in decoded_from.iter().rev().copied() { + body = decoder_for(body, coding).await?; + } + + let mut decoded = DecodedHttpResponse { status, final_url, headers, body, decoded_from, original_content_length }; + if decoded.was_content_decoded() { + normalize_headers_after_content_decoding(&mut decoded.headers); + } else if has_content_encoding { + decoded.headers.remove(CONTENT_ENCODING); + } + + Ok(decoded) +} + +pub(crate) async fn read_to_end_limited(reader: &mut R, max_bytes: usize) -> Result, ContentBodyReadError> +where + R: AsyncRead + Unpin, +{ + let probe_limit = max_bytes.checked_add(1); + let mut result = Vec::with_capacity(max_bytes.min(8 * 1024)); + let mut buffer = [0_u8; 8 * 1024]; + + loop { + let read_len = probe_limit.map_or(buffer.len(), |limit| limit.saturating_sub(result.len()).min(buffer.len())); + if read_len == 0 { + return Err(ContentBodyReadError::LimitExceeded { limit: max_bytes }); + } + + let read = reader.read(&mut buffer[..read_len]).await?; + if read == 0 { + return Ok(result); + } + result.extend_from_slice(&buffer[..read]); + + if result.len() > max_bytes { + return Err(ContentBodyReadError::LimitExceeded { limit: max_bytes }); + } + } +} + +pub(crate) async fn read_utf8_limited(reader: &mut R, max_bytes: usize) -> Result +where + R: AsyncRead + Unpin, +{ + let bytes = read_to_end_limited(reader, max_bytes).await?; + String::from_utf8(bytes).map_err(|error| { + let utf8_error = error.utf8_error(); + ContentBodyReadError::InvalidUtf8 { valid_up_to: utf8_error.valid_up_to(), error_len: utf8_error.error_len() } + }) +} + +pub(crate) fn normalize_headers_after_content_decoding(headers: &mut HeaderMap) { + for name in [ + CONTENT_ENCODING, + CONTENT_LENGTH, + CONTENT_RANGE, + ACCEPT_RANGES, + ETAG, + HeaderName::from_static("content-md5"), + HeaderName::from_static("digest"), + HeaderName::from_static("content-digest"), + HeaderName::from_static("repr-digest"), + TRANSFER_ENCODING, + ] { + headers.remove(name); + } + + remove_accept_encoding_from_vary(headers); +} + +pub(crate) fn content_decoding_error_from_io(error: &io::Error) -> Option<&ContentDecodingIoError> { + find_content_decoding_error(error.get_ref()?) +} + +async fn decoder_for(mut reader: DynReader, coding: ContentCoding) -> Result { + let reader = match coding { + ContentCoding::Gzip => tagged_decoder(GzipDecoder::new(BufReader::new(reader)), coding), + ContentCoding::Brotli => tagged_decoder(BrotliDecoder::new(BufReader::new(reader)), coding), + ContentCoding::Zstd => tagged_decoder(ZstdDecoder::new(BufReader::new(reader)), coding), + ContentCoding::Deflate => { + let (replayed_reader, prefix) = inspect_prefix(reader, 2).await?; + reader = replayed_reader; + if is_zlib_header(&prefix) { + tagged_decoder(ZlibDecoder::new(BufReader::new(reader)), coding) + } else { + tagged_decoder(DeflateDecoder::new(BufReader::new(reader)), coding) + } + } + }; + Ok(reader) +} + +fn tagged_decoder(reader: R, coding: ContentCoding) -> DynReader +where + R: AsyncRead + Send + Unpin + 'static, +{ + Box::pin(ContentDecodingReader { inner: reader, coding }) +} + +async fn inspect_prefix(mut reader: DynReader, max_len: usize) -> io::Result<(DynReader, Vec)> { + let mut prefix = Vec::with_capacity(max_len); + while prefix.len() < max_len { + let mut chunk = [0_u8; MAX_MAGIC_PREFIX_LEN]; + let remaining = max_len - prefix.len(); + let read = reader.read(&mut chunk[..remaining]).await?; + if read == 0 { + break; + } + prefix.extend_from_slice(&chunk[..read]); + } + + let replayed = std::io::Cursor::new(prefix.clone()).chain(reader); + Ok((Box::pin(replayed), prefix)) +} + +fn coding_from_known_manifest_magic(prefix: &[u8]) -> Option { + coding_from_legacy_text_magic(prefix).or_else(|| prefix.starts_with(&ZSTD_MAGIC).then_some(ContentCoding::Zstd)) +} + +fn coding_from_legacy_text_magic(prefix: &[u8]) -> Option { + if is_gzip(prefix) { + Some(ContentCoding::Gzip) + } else { + is_zlib_header(prefix).then_some(ContentCoding::Deflate) + } +} + +fn remove_accept_encoding_from_vary(headers: &mut HeaderMap) { + let values = headers.get_all(VARY).iter().cloned().collect::>(); + if values.is_empty() { + return; + } + headers.remove(VARY); + + for value in values { + let original_value = value.clone(); + let Ok(value) = value.to_str() else { + headers.append(VARY, original_value); + continue; + }; + let remaining = value + .split(',') + .map(str::trim) + .filter(|token| !token.eq_ignore_ascii_case("accept-encoding")) + .collect::>(); + if !remaining.is_empty() { + match HeaderValue::from_str(&remaining.join(", ")) { + Ok(value) => { + headers.append(VARY, value); + } + Err(_) => { + headers.append(VARY, original_value); + } + } + } + } +} + +fn find_content_decoding_error<'a>(mut error: &'a (dyn StdError + 'static)) -> Option<&'a ContentDecodingIoError> { + loop { + if let Some(error) = error.downcast_ref::() { + return Some(error); + } + error = error.source()?; + } +} + +struct ContentDecodingReader { + inner: R, + coding: ContentCoding, +} + +impl AsyncRead for ContentDecodingReader +where + R: AsyncRead + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + match Pin::new(&mut self.inner).poll_read(context, buffer) { + Poll::Ready(Err(error)) + if content_decoding_error_from_io(&error).is_none() && !is_http_body_transport_error(&error) => + { + let kind = error.kind(); + Poll::Ready(Err(io::Error::new(kind, ContentDecodingIoError { coding: self.coding }))) + } + result => result, + } + } +} + +pub(crate) fn is_http_body_transport_error(error: &io::Error) -> bool { + let Some(source) = error.get_ref() else { + return false; + }; + let mut source: &(dyn StdError + 'static) = source; + loop { + if source.downcast_ref::().is_some() { + return true; + } + let Some(next) = source.source() else { + return false; + }; + source = next; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZlibEncoder, ZstdEncoder}; + use reqwest::redirect::Policy; + use std::io::Cursor; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:3\nsegment.ts\n"; + + #[test] + fn content_coding_parses_all_headers_tokens_aliases_and_identity() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_ENCODING, HeaderValue::from_static("gzip, identity")); + headers.append(CONTENT_ENCODING, HeaderValue::from_static("BR, x-GZiP, deflate, zstd")); + + assert_eq!( + parse_content_codings(&headers).expect("valid codings"), + vec![ + ContentCoding::Gzip, + ContentCoding::Brotli, + ContentCoding::Gzip, + ContentCoding::Deflate, + ContentCoding::Zstd, + ] + ); + } + + #[test] + fn content_coding_rejects_empty_invalid_and_unsupported_tokens() { + for value in [HeaderValue::from_static("gzip,"), HeaderValue::from_static("gzip; level=1")] { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_ENCODING, value); + assert!(matches!(parse_content_codings(&headers), Err(ContentCodingError::InvalidHeader))); + } + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_ENCODING, HeaderValue::from_bytes(&[0xff]).expect("opaque header value")); + assert!(matches!(parse_content_codings(&headers), Err(ContentCodingError::InvalidHeader))); + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_ENCODING, HeaderValue::from_static("compress")); + assert!(matches!( + parse_content_codings(&headers), + Err(ContentCodingError::Unsupported(coding)) if coding == "compress" + )); + } + + #[test] + fn content_encoding_token_parser_preserves_unknown_valid_tokens_and_order() { + let mut headers = HeaderMap::new(); + headers.append(CONTENT_ENCODING, HeaderValue::from_static("gzip, X-Provider-Coding")); + headers.append(CONTENT_ENCODING, HeaderValue::from_static("compress")); + + assert_eq!( + parse_content_encoding_tokens(&headers).expect("valid HTTP tokens"), + vec!["gzip".to_owned(), "X-Provider-Coding".to_owned(), "compress".to_owned()] + ); + assert!(matches!( + parse_content_codings(&headers), + Err(ContentCodingError::Unsupported(coding)) if coding == "X-Provider-Coding" + )); + } + + #[test] + fn content_coding_identity_policy_overrides_existing_value_only_when_requested() { + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip, br")); + + apply_outbound_content_coding_policy(&mut headers, OutboundContentCodingPolicy::Inherit); + assert_eq!(headers[header::ACCEPT_ENCODING], "gzip, br"); + + apply_outbound_content_coding_policy(&mut headers, OutboundContentCodingPolicy::Identity); + assert_eq!(headers[header::ACCEPT_ENCODING], "identity"); + } + + #[tokio::test] + async fn content_coding_decodes_all_declared_codings_and_deflate_wrappers() { + let cases = [ + ("gzip", Encoding::Gzip), + ("x-gzip", Encoding::Gzip), + ("deflate", Encoding::Zlib), + ("deflate", Encoding::RawDeflate), + ("br", Encoding::Brotli), + ("zstd", Encoding::Zstd), + ]; + + for (header_value, encoding) in cases { + let encoded = encode(MANIFEST, encoding).await; + let response = local_response(StatusCode::OK, &[("Content-Encoding", header_value)], encoded).await; + let mut decoded = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly) + .await + .expect("decoder setup"); + + assert_eq!( + read_to_end_limited(&mut decoded.body, 1024).await.expect("decoded body"), + MANIFEST, + "failed for {header_value} / {encoding:?}" + ); + assert!(!decoded.headers.contains_key(CONTENT_ENCODING)); + } + } + + #[tokio::test] + async fn content_coding_decodes_multiple_header_lines_in_reverse_application_order() { + let gzip = encode(MANIFEST, Encoding::Gzip).await; + let brotli_over_gzip = encode(&gzip, Encoding::Brotli).await; + let encoded_length = brotli_over_gzip.len() as u64; + let response = local_response( + StatusCode::OK, + &[("Content-Encoding", "gzip"), ("Content-Encoding", "br")], + brotli_over_gzip, + ) + .await; + + let mut decoded = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("decoder setup"); + assert_eq!(decoded.decoded_from, vec![ContentCoding::Gzip, ContentCoding::Brotli]); + assert_eq!( + decoded.content_coding_observation(), + Some(ContentCodingObservation { + content_encoding: "multiple", + status: StatusCode::OK, + content_length: Some(encoded_length), + }) + ); + assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("decoded body"), MANIFEST); + } + + #[tokio::test] + async fn content_coding_decodes_comma_separated_codings_in_reverse_application_order() { + let gzip = encode(MANIFEST, Encoding::Gzip).await; + let brotli_over_gzip = encode(&gzip, Encoding::Brotli).await; + let response = local_response(StatusCode::OK, &[("Content-Encoding", "gzip, br")], brotli_over_gzip).await; + + let mut decoded = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("decoder setup"); + assert_eq!(decoded.decoded_from, vec![ContentCoding::Gzip, ContentCoding::Brotli]); + assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("decoded body"), MANIFEST); + } + + #[tokio::test] + async fn content_coding_declared_value_wins_over_contradicting_magic() { + for detection in + [ContentCodingDetection::DeclaredOrLegacyTextMagic, ContentCodingDetection::DeclaredOrKnownHlsManifestMagic] + { + let gzip = encode(MANIFEST, Encoding::Gzip).await; + let response = local_response(StatusCode::OK, &[("Content-Encoding", "br")], gzip).await; + let mut decoded = decode_response_to_identity(response, detection).await.expect("lazy decoder setup"); + + let error = read_to_end_limited(&mut decoded.body, 1024) + .await + .expect_err("gzip bytes must not override declared Brotli"); + let ContentBodyReadError::Io(error) = error else { + panic!("expected decoder I/O error"); + }; + assert_eq!( + content_decoding_error_from_io(&error).expect("typed decoder error").coding, + ContentCoding::Brotli + ); + } + } + + #[tokio::test] + async fn content_coding_legacy_text_magic_decodes_only_headerless_gzip_and_zlib() { + for encoding in [Encoding::Gzip, Encoding::Zlib] { + let encoded = encode(MANIFEST, encoding).await; + let response = local_response(StatusCode::OK, &[], encoded).await; + let mut decoded = decode_response_to_identity(response, ContentCodingDetection::DeclaredOrLegacyTextMagic) + .await + .expect("decoder setup"); + + assert_eq!( + read_to_end_limited(&mut decoded.body, 1024).await.expect("magic-decoded body"), + MANIFEST, + "failed for {encoding:?}" + ); + assert!(decoded.was_content_decoded()); + } + + for encoding in [Encoding::RawDeflate, Encoding::Brotli, Encoding::Zstd] { + let encoded = encode(MANIFEST, encoding).await; + let response = local_response(StatusCode::OK, &[], encoded.clone()).await; + let mut untouched = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrLegacyTextMagic) + .await + .expect("identity setup"); + + assert!(!untouched.was_content_decoded(), "must not guess {encoding:?}"); + assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded); + } + } + + #[tokio::test] + async fn content_coding_hls_manifest_magic_supports_only_known_signatures() { + for encoding in [Encoding::Gzip, Encoding::Zlib, Encoding::Zstd] { + let encoded = encode(MANIFEST, encoding).await; + let response = local_response(StatusCode::OK, &[], encoded).await; + let mut decoded = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrKnownHlsManifestMagic) + .await + .expect("decoder setup"); + assert_eq!( + read_to_end_limited(&mut decoded.body, 1024).await.expect("magic-decoded body"), + MANIFEST, + "failed for {encoding:?}" + ); + } + + let encoded_binary = encode(b"not a manifest", Encoding::Gzip).await; + let response = local_response(StatusCode::OK, &[], encoded_binary.clone()).await; + let mut untouched = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("identity setup"); + assert!(untouched.decoded_from.is_empty()); + assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded_binary); + } + + #[tokio::test] + async fn content_coding_header_presence_disables_manifest_magic_even_for_identity() { + let encoded = encode(MANIFEST, Encoding::Gzip).await; + let response = local_response(StatusCode::OK, &[("Content-Encoding", "identity")], encoded.clone()).await; + let mut decoded = + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrKnownHlsManifestMagic) + .await + .expect("identity setup"); + + assert!(decoded.decoded_from.is_empty()); + assert_eq!(decoded.content_coding_observation(), None); + assert!(!decoded.headers.contains_key(CONTENT_ENCODING)); + assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("identity body"), encoded); + } + + #[tokio::test] + async fn content_coding_rejects_encoded_partial_content_but_preserves_identity_range_headers() { + let encoded = encode(MANIFEST, Encoding::Gzip).await; + let response = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/20")], + encoded, + ) + .await; + assert!(matches!( + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await, + Err(ContentCodingError::EncodedPartialContent) + )); + + let response = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "identity"), ("Content-Range", "bytes 0-2/10")], + b"abc".to_vec(), + ) + .await; + let decoded = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly) + .await + .expect("identity partial response"); + assert_eq!(decoded.status, StatusCode::PARTIAL_CONTENT); + assert_eq!(decoded.headers[CONTENT_RANGE], "bytes 0-2/10"); + } + + #[tokio::test] + async fn content_coding_normalizes_only_representation_dependent_headers() { + let encoded = encode(MANIFEST, Encoding::Gzip).await; + let response = local_response( + StatusCode::OK, + &[ + ("Content-Encoding", "gzip"), + ("Content-Range", "bytes 0-9/10"), + ("Accept-Ranges", "bytes"), + ("ETag", "origin-tag"), + ("Content-MD5", "abc"), + ("Digest", "sha-256=abc"), + ("Content-Digest", "sha-256=:abc:"), + ("Repr-Digest", "sha-256=:abc:"), + ("Vary", "Accept-Encoding, Origin"), + ("Vary", "*"), + ("Set-Cookie", "session=secret"), + ], + encoded, + ) + .await; + let decoded = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly) + .await + .expect("decoded response"); + + for name in [ + CONTENT_ENCODING, + CONTENT_LENGTH, + CONTENT_RANGE, + ACCEPT_RANGES, + ETAG, + HeaderName::from_static("content-md5"), + HeaderName::from_static("digest"), + HeaderName::from_static("content-digest"), + HeaderName::from_static("repr-digest"), + TRANSFER_ENCODING, + ] { + assert!(!decoded.headers.contains_key(&name), "stale header {name}"); + } + let vary = + decoded.headers.get_all(VARY).iter().map(|value| value.to_str().expect("ASCII Vary")).collect::>(); + assert_eq!(vary, vec!["Origin", "*"]); + assert_eq!(decoded.headers[header::SET_COOKIE], "session=secret"); + } + + #[test] + fn content_coding_normalizer_removes_transfer_encoding_but_keeps_safe_metadata() { + let mut headers = HeaderMap::new(); + headers.insert(TRANSFER_ENCODING, HeaderValue::from_static("chunked")); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/vnd.apple.mpegurl")); + + normalize_headers_after_content_decoding(&mut headers); + + assert!(!headers.contains_key(TRANSFER_ENCODING)); + assert_eq!(headers[header::CONTENT_TYPE], "application/vnd.apple.mpegurl"); + } + + #[test] + fn content_coding_normalizer_preserves_opaque_vary_without_panicking() { + let opaque_vary = HeaderValue::from_bytes(&[0xff]).expect("opaque header value"); + let mut headers = HeaderMap::new(); + headers.insert(VARY, opaque_vary.clone()); + + normalize_headers_after_content_decoding(&mut headers); + + assert_eq!(headers[VARY], opaque_vary); + } + + #[tokio::test] + async fn content_coding_surfaces_typed_errors_for_truncated_streams() { + for (encoding, coding) in [ + (Encoding::Gzip, ContentCoding::Gzip), + (Encoding::Brotli, ContentCoding::Brotli), + (Encoding::Zstd, ContentCoding::Zstd), + ] { + let mut encoded = encode(MANIFEST, encoding).await; + encoded.truncate(encoded.len() / 2); + let response = + local_response(StatusCode::OK, &[("Content-Encoding", encoding.header_value())], encoded).await; + let mut decoded = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly) + .await + .expect("lazy decoder setup"); + + let error = read_to_end_limited(&mut decoded.body, 1024).await.expect_err("truncated stream"); + let ContentBodyReadError::Io(error) = error else { + panic!("expected decoder I/O error for {encoding:?}"); + }; + assert_eq!(content_decoding_error_from_io(&error).expect("typed decoder error").coding, coding); + } + } + + #[tokio::test] + async fn content_coding_limit_and_utf8_errors_are_distinct() { + let mut exact = Cursor::new(b"abcd".to_vec()); + assert_eq!(read_to_end_limited(&mut exact, 4).await.expect("exact limit"), b"abcd"); + + let mut oversized = Cursor::new(b"abcde".to_vec()); + assert!(matches!( + read_to_end_limited(&mut oversized, 4).await, + Err(ContentBodyReadError::LimitExceeded { limit: 4 }) + )); + + let mut invalid_utf8 = Cursor::new(vec![b'a', 0xff]); + assert!(matches!( + read_utf8_limited(&mut invalid_utf8, 10).await, + Err(ContentBodyReadError::InvalidUtf8 { valid_up_to: 1, error_len: Some(1) }) + )); + } + + #[derive(Debug, Clone, Copy)] + enum Encoding { + Gzip, + Zlib, + RawDeflate, + Brotli, + Zstd, + } + + impl Encoding { + const fn header_value(self) -> &'static str { + match self { + Self::Gzip => "gzip", + Self::Zlib | Self::RawDeflate => "deflate", + Self::Brotli => "br", + Self::Zstd => "zstd", + } + } + } + + async fn encode(input: &[u8], encoding: Encoding) -> Vec { + let reader = BufReader::new(Cursor::new(input.to_vec())); + let mut result = Vec::new(); + match encoding { + Encoding::Gzip => GzipEncoder::new(reader).read_to_end(&mut result).await.expect("gzip encode"), + Encoding::Zlib => ZlibEncoder::new(reader).read_to_end(&mut result).await.expect("zlib encode"), + Encoding::RawDeflate => { + DeflateEncoder::new(reader).read_to_end(&mut result).await.expect("raw deflate encode") + } + Encoding::Brotli => BrotliEncoder::new(reader).read_to_end(&mut result).await.expect("Brotli encode"), + Encoding::Zstd => ZstdEncoder::new(reader).read_to_end(&mut result).await.expect("Zstandard encode"), + }; + result + } + + async fn local_response(status: StatusCode, headers: &[(&str, &str)], body: Vec) -> reqwest::Response { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test origin"); + let address = listener.local_addr().expect("test origin address"); + let has_content_length = headers.iter().any(|(name, _)| name.eq_ignore_ascii_case("content-length")); + let owned_headers = + headers.iter().map(|(name, value)| ((*name).to_owned(), (*value).to_owned())).collect::>(); + + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept test request"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut buffer).await.expect("read test request"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + } + + let mut response = format!("HTTP/1.1 {} Test\r\nConnection: close\r\n", status.as_u16()); + if !has_content_length { + response.push_str(&format!("Content-Length: {}\r\n", body.len())); + } + for (name, value) in owned_headers { + response.push_str(&name); + response.push_str(": "); + response.push_str(&value); + response.push_str("\r\n"); + } + response.push_str("\r\n"); + socket.write_all(response.as_bytes()).await.expect("write test response head"); + socket.write_all(&body).await.expect("write test response body"); + }); + + reqwest::Client::builder() + .no_proxy() + .redirect(Policy::none()) + .build() + .expect("test client") + .get(format!("http://{address}/manifest.m3u8")) + .send() + .await + .expect("fetch test response") + } +} diff --git a/backend/src/utils/network/mod.rs b/backend/src/utils/network/mod.rs index 1c026709f..ab47a1864 100644 --- a/backend/src/utils/network/mod.rs +++ b/backend/src/utils/network/mod.rs @@ -1,7 +1,14 @@ +use std::pin::Pin; +use tokio::io::AsyncRead; + +pub(crate) mod content_coding; pub mod epg; pub mod ip_checker; pub mod m3u; pub mod request; pub mod xtream; +/// Type-erased asynchronous reader used by the existing network download stack. +pub type DynReader = Pin>; + pub use request::format_http_status; diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 3930bfcc2..e440c4974 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -1,64 +1,114 @@ +pub use super::DynReader; use crate::{ - api::model::{persist_pipe_stream::tee_dyn_reader, AppState, STREAM_IDLE_TIMEOUT}, + api::model::{ + log_hls_origin_content_coding, persist_pipe_stream::tee_dyn_reader, AppState, HlsOriginContentCodingObjectKind, + HlsOriginContentCodingSource, STREAM_IDLE_TIMEOUT, + }, model::{ resolve_provider_scheme_url_with_provider_index, AppConfig, Config, ConfigInput, ConfigProvider, InputSource, ResourceRetryConfig, ReverseProxyDisabledHeaderConfig, }, utils::{ async_file_reader, async_file_writer, - compression::compression_utils::{is_deflate, is_gzip}, + compression::compression_utils::is_gzip, + content_coding::{ + apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, + is_http_body_transport_error, read_utf8_limited, ContentBodyReadError, ContentCodingDetection, + ContentCodingError, OutboundContentCodingPolicy, + }, debug_if_enabled, get_file_path, persist_file, }, }; -use futures::{StreamExt, TryStreamExt}; +use futures::StreamExt; use log::{debug, error, log_enabled, trace, warn, Level}; use regex::Regex; use reqwest::{ - header::{HeaderMap, HeaderName, HeaderValue, CONTENT_ENCODING, HOST}, + header::{HeaderMap, HeaderName, HeaderValue, HOST}, redirect::Policy, StatusCode, }; use shared::{ + defaults::DEFAULT_USER_AGENT, error::{string_to_io_error, TuliproxError}, model::{format_elapsed_time, InputFetchMethod, OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, - utils::{ - filter_request_header, human_readable_byte_size, sanitize_sensitive_info, CONTENT_TYPE_JSON, ENCODING_DEFLATE, - ENCODING_GZIP, - }, - defaults::{DEFAULT_USER_AGENT} + utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, CONTENT_TYPE_JSON}, }; use std::{ collections::{HashMap, HashSet}, io::{Error, ErrorKind}, net::{IpAddr, SocketAddr}, path::{Path, PathBuf}, - pin::Pin, sync::{Arc, Once}, time::Duration, }; use tokio::{ fs::File, - io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt}, + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}, time::sleep, }; -use tokio_util::io::StreamReader; use url::Url; static PROXY_DIAGNOSTICS_ONCE: Once = Once::new(); +/// Options applied at the final boundary of every physical request attempt. #[derive(Debug, Clone, Copy, Default)] pub struct RequestFetchOptions { pub attempt_idle_timeout: Option, + content_coding: OutboundContentCodingPolicy, + resource_retry: ResourceRetryExecution, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum ResourceRetryExecution { + #[default] + Configured, + ProviderFailoverOnly, } impl RequestFetchOptions { pub fn with_attempt_idle_timeout(timeout: Duration) -> Self { - Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))) } + Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))), ..Self::default() } + } + + pub(crate) const fn with_content_coding(mut self, content_coding: OutboundContentCodingPolicy) -> Self { + self.content_coding = content_coding; + self + } + + /// Leaves bounded provider failover intact while assigning retry rounds to the caller. + pub(crate) const fn without_resource_retries(mut self) -> Self { + self.resource_retry = ResourceRetryExecution::ProviderFailoverOnly; + self } fn attempt_idle_timeout_or_default(self) -> Duration { self.attempt_idle_timeout.unwrap_or_else(|| Duration::from_secs(STREAM_IDLE_TIMEOUT)) } + + const fn uses_provider_failover_only(self) -> bool { + matches!(self.resource_retry, ResourceRetryExecution::ProviderFailoverOnly) + } +} + +fn apply_request_fetch_options(request: &mut reqwest::Request, options: RequestFetchOptions) { + if let Some(timeout) = options.attempt_idle_timeout { + *request.timeout_mut() = Some(timeout); + } + apply_outbound_content_coding_policy(request.headers_mut(), options.content_coding); +} + +fn prepare_physical_request_attempt( + request_builder: reqwest::RequestBuilder, + target: &AttemptTarget, + options: RequestFetchOptions, +) -> Result<(reqwest::Client, reqwest::Request), std::io::Error> { + let (base_client, request_result) = request_builder.build_split(); + let mut request = request_result.map_err(|error| { + string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(error.to_string().as_str()))) + })?; + apply_attempt_to_request(&mut request, target)?; + apply_request_fetch_options(&mut request, options); + Ok((base_client, request)) } #[derive(Debug, Clone, Copy, Default)] @@ -258,11 +308,8 @@ fn resolve_attempt_target_with_dns_mode( return target; } - let connect_ip = if preview_dns_selection { - provider.preview_ip_for_host(host) - } else { - provider.select_ip_for_host(host) - }; + let connect_ip = + if preview_dns_selection { provider.preview_ip_for_host(host) } else { provider.select_ip_for_host(host) }; let Some(connect_ip) = connect_ip else { return target; }; @@ -351,7 +398,8 @@ fn rotate_to_next_provider_url( start_provider_index: usize, reason: &str, ) -> bool { - let Some(next_index) = next_provider_url_index(*provider_url_index, provider.urls.len(), start_provider_index) else { + let Some(next_index) = next_provider_url_index(*provider_url_index, provider.urls.len(), start_provider_index) + else { return false; }; @@ -433,10 +481,7 @@ fn should_try_next_ip_on_connect_error( total_ips > attempted_ips.len() } -fn apply_attempt_to_request( - request: &mut reqwest::Request, - target: &AttemptTarget, -) -> Result<(), std::io::Error> { +fn apply_attempt_to_request(request: &mut reqwest::Request, target: &AttemptTarget) -> Result<(), std::io::Error> { if request.url().as_str() != target.effective_url.as_str() { *request.url_mut() = target.effective_url.clone(); } @@ -481,34 +526,30 @@ async fn execute_attempt_request( } /// Response returned after applying provider URL failover without applying the generic resource retry policy. -pub struct ProviderFailoverResponse { - pub response: reqwest::Response, - pub provider_url_index: Option, +pub(crate) struct ProviderFailoverResponse { + pub(crate) response: reqwest::Response, + pub(crate) provider_url_index: Option, } -/// Sends one logical request while allowing one bounded provider URL failover cycle. -/// -/// This reuses provider URL resolution, provider DNS handling, global client/proxy configuration and failover status -/// classification, but deliberately does not apply `reverse_proxy.resource_retry` attempts or backoff. #[allow(clippy::too_many_lines)] -pub async fn send_with_provider_failover_only( +async fn send_with_provider_failover_only_with_options( app_config: &Arc, url: &Url, provider: Option<&Arc>, allow_redirects: bool, + options: RequestFetchOptions, mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, ) -> Result { - let failover_patterns = app_config - .config - .load() - .reverse_proxy - .as_ref() - .map_or_else(|| ResourceRetryConfig::default().failover_redirect_patterns, |rp| { - rp.resource_retry.failover_redirect_patterns.clone() - }); + let failover_patterns = app_config.config.load().reverse_proxy.as_ref().map_or_else( + || ResourceRetryConfig::default().failover_redirect_patterns, + |rp| rp.resource_retry.failover_redirect_patterns.clone(), + ); let start_provider_index = provider_start_index(provider); let mut provider_url_index = start_provider_index; + let idle_timeout = options.attempt_idle_timeout_or_default(); + let idle = sleep(idle_timeout); + tokio::pin!(idle); 'provider_loop: loop { let mut attempted_dns_ips = HashSet::new(); @@ -528,14 +569,43 @@ pub async fn send_with_provider_failover_only( } } - let request_builder = send(&attempt_target.request_url); - let (base_client, request_result) = request_builder.build_split(); - let mut request = request_result.map_err(|err| { - string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(err.to_string().as_str()))) - })?; - apply_attempt_to_request(&mut request, &attempt_target)?; + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + let (base_client, request) = + prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; - match execute_attempt_request(app_config, base_client, request, &attempt_target).await { + tokio::select! { + () = &mut idle => { + if should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) { + continue 'ip_loop; + } + + let last_provider_failure = format!( + "idle timeout while trying {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()) + ); + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "idle timeout", + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + + return Err(Error::new( + ErrorKind::TimedOut, + format!("Request timed out: {}", sanitize_sensitive_info(url.as_str())), + )); + } + result = execute_attempt_request(app_config, base_client, request, &attempt_target) => match result { Ok(response) => { let status = response.status(); if allow_redirects && status.is_redirection() { @@ -589,7 +659,7 @@ pub async fn send_with_provider_failover_only( response, provider_url_index: provider.map(|_| provider_url_index), }); - } + }, Err(err) => { if (err.is_timeout() || err.is_connect()) && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) @@ -622,10 +692,15 @@ pub async fn send_with_provider_failover_only( } } - return Err(string_to_io_error(format!( - "Request error: {}", - sanitize_sensitive_info(err.to_string().as_str()) - ))); + let message = format!("Request error: {}", sanitize_sensitive_info(err.to_string().as_str())); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, message) + } else if err.is_connect() { + Error::new(ErrorKind::ConnectionRefused, message) + } else { + string_to_io_error(message) + }); + }, } } } @@ -700,7 +775,7 @@ pub async fn send_with_retry_and_provider_policy( } #[allow(clippy::too_many_lines)] -pub async fn send_with_retry_and_provider_policy_with_options( +async fn send_with_retry_and_provider_policy_with_options( app_config: &Arc, url: &Url, // Used primarily for logging/context provider: Option<&Arc>, @@ -723,7 +798,7 @@ pub async fn send_with_retry_and_provider_policy_with_options( } #[allow(clippy::too_many_lines)] -pub async fn send_with_retry_and_provider_policy_with_options_result( +async fn send_with_retry_and_provider_policy_with_options_result( app_config: &Arc, url: &Url, // Used primarily for logging/context provider: Option<&Arc>, @@ -732,6 +807,18 @@ pub async fn send_with_retry_and_provider_policy_with_options_result( options: RequestFetchOptions, mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, ) -> Result { + if options.uses_provider_failover_only() { + return send_with_provider_failover_only_with_options( + app_config, + url, + provider, + allow_redirects, + options, + send, + ) + .await; + } + let config = app_config.config.load(); let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( || { @@ -777,15 +864,8 @@ pub async fn send_with_retry_and_provider_policy_with_options_result( // Reset the idle timer for a new attempt idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); - let request_builder = send(&attempt_target.request_url); - let (base_client, request_result) = request_builder.build_split(); - let mut request = request_result.map_err(|err| { - string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(&err.to_string()))) - })?; - apply_attempt_to_request(&mut request, &attempt_target)?; - if let Some(timeout) = options.attempt_idle_timeout { - *request.timeout_mut() = Some(timeout); - } + let (base_client, request) = + prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; tokio::select! { () = &mut idle => { @@ -830,7 +910,13 @@ pub async fn send_with_retry_and_provider_policy_with_options_result( } } - return Err(string_to_io_error(format!("Request timed out and no retries left: {}", sanitize_sensitive_info(url.as_str())))); + return Err(Error::new( + ErrorKind::TimedOut, + format!( + "Request timed out and no retries left: {}", + sanitize_sensitive_info(url.as_str()) + ), + )); } result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { @@ -959,7 +1045,15 @@ pub async fn send_with_retry_and_provider_policy_with_options_result( } } - return Err(string_to_io_error(format!("Request error: {}", sanitize_sensitive_info(&err.to_string())))); + let error_message = format!( + "Request error: {}", + sanitize_sensitive_info(&err.to_string()) + ); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, error_message) + } else { + string_to_io_error(error_message) + }); } } } @@ -980,7 +1074,8 @@ pub async fn send_with_retry_and_provider_policy_with_options_result( } if max_provider_attempts > 0 { - let last_failure = last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); + let last_failure = + last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); log_provider_cycle_exhausted( current_provider.as_ref(), start_provider_index, @@ -1026,7 +1121,7 @@ fn prepare_input_request_headers( } #[allow(clippy::implicit_hasher)] -pub async fn send_input_with_retry_and_provider_policy_with_options_result( +pub(crate) async fn send_input_with_retry_and_provider_policy_with_options_result( app_config: &Arc, client: &reqwest::Client, input: &InputSource, @@ -1058,7 +1153,7 @@ pub async fn send_input_with_retry_and_provider_policy_with_options_result( } #[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::implicit_hasher)] -pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( +pub(crate) async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( app_config: &Arc, client: &reqwest::Client, input: &InputSource, @@ -1068,17 +1163,20 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and options: RequestFetchOptions, ) -> Result { let config = app_config.config.load(); - let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( - || { - let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); - (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) - }, - |rp| { - let (a, b, c) = rp.resource_retry.get_retry_values(); - (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) - }, - ); + let (configured_max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = + config.reverse_proxy.as_ref().map_or_else( + || { + let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); + (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) + }, + |rp| { + let (a, b, c) = rp.resource_retry.get_retry_values(); + (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) + }, + ); drop(config); + let provider_failover_only = options.uses_provider_failover_only(); + let max_attempts = if provider_failover_only { 1 } else { configured_max_attempts }; let (base_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); let provider = input.get_provider(); @@ -1098,7 +1196,8 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and let mut attempted_dns_ips = HashSet::new(); 'redirect_loop: loop { - let attempt_target = resolve_attempt_target_at_provider_index(¤t_url, provider, provider_url_index); + let attempt_target = + resolve_attempt_target_at_provider_index(¤t_url, provider, provider_url_index); if log_enabled!(Level::Debug) { if let Some(current_provider) = provider { let attempt_target_log = format_request_target_for_logging(&attempt_target); @@ -1122,14 +1221,8 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and None, default_user_agent.as_deref(), ); - let (base_client, request_result) = request_builder.build_split(); - let mut request = request_result.map_err(|err| { - string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(err.to_string().as_str()))) - })?; - apply_attempt_to_request(&mut request, &attempt_target)?; - if let Some(timeout) = options.attempt_idle_timeout { - *request.timeout_mut() = Some(timeout); - } + let (base_client, request) = + prepare_physical_request_attempt(request_builder, &attempt_target, options)?; tokio::select! { () = &mut idle => { @@ -1164,7 +1257,13 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and continue 'attempt_loop; } - return Err(string_to_io_error(format!("Request timed out and no retries left: {}", sanitize_sensitive_info(url.as_str())))); + return Err(Error::new( + ErrorKind::TimedOut, + format!( + "Request timed out and no retries left: {}", + sanitize_sensitive_info(url.as_str()) + ), + )); } result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { @@ -1263,6 +1362,13 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and } } + if provider_failover_only { + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + return Err(string_to_io_error(format!( "Request failed ({}): {}", format_http_status(status), @@ -1316,10 +1422,17 @@ pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and } } - return Err(string_to_io_error(format!( + let error_message = format!( "Request error: {}", sanitize_sensitive_info(err.to_string().as_str()) - ))); + ); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, error_message) + } else if err.is_connect() { + Error::new(ErrorKind::ConnectionRefused, error_message) + } else { + string_to_io_error(error_message) + }); } } } @@ -1377,13 +1490,7 @@ pub async fn get_input_epg_content_as_file( input: &ConfigInput, request: InputEpgFileRequest<'_>, ) -> Result { - let InputEpgFileRequest { - headers, - storage_dir, - url: url_str, - persist_path, - max_bytes, - } = request; + let InputEpgFileRequest { headers, storage_dir, url: url_str, persist_path, max_bytes } = request; debug_if_enabled!( "getting input epg content storage_dir: {}, url: {}", storage_dir, @@ -1450,11 +1557,7 @@ pub async fn get_input_text_content( match download_text_content(&app_state.app_config, client, input, None, persist_filepath, false).await { Ok((content, _response_url)) => Ok(content), Err(e) => { - error!( - "Failed to download input '{}': {}", - input.name, - sanitize_sensitive_info(&e.to_string()) - ); + error!("Failed to download input '{}': {}", input.name, sanitize_sensitive_info(&e.to_string())); Err(TuliproxError::RepositoryNetwork(format!( "Failed to download input '{}': {}", input.name, @@ -1470,7 +1573,11 @@ pub async fn get_input_text_content( let to_file = &persist_file_value; if let Err(e) = tokio::fs::copy(&filepath, to_file).await { error!("can't persist to: {} => {}", to_file.to_str().unwrap_or("?"), e); - return Err(TuliproxError::RepositoryNetwork(format!("Failed to persist: {} => {}", to_file.to_str().unwrap_or("?"), e))); + return Err(TuliproxError::RepositoryNetwork(format!( + "Failed to persist: {} => {}", + to_file.to_str().unwrap_or("?"), + e + ))); } } @@ -1514,11 +1621,7 @@ pub async fn get_input_text_content_as_stream( match download_text_content_as_stream(app_config, client, input, persist_filepath).await { Ok((content, _response_url)) => Ok(content), Err(e) => { - error!( - "Failed to download input '{}': {}", - input.name, - sanitize_sensitive_info(&e.to_string()) - ); + error!("Failed to download input '{}': {}", input.name, sanitize_sensitive_info(&e.to_string())); Err(TuliproxError::RepositoryNetwork(format!( "Failed to download input '{}': {}", input.name, @@ -1540,7 +1643,7 @@ pub async fn get_input_text_content_as_stream( debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); })), ) - .await; + .await; Some(tee) } else { Some(content) @@ -1847,35 +1950,160 @@ pub async fn get_remote_content_as_file_with_options( Ok(file_path.to_path_buf()) } -pub type DynReader = Pin>; +/// Controls decoding and bounded consumption of a fully buffered text response. +#[derive(Debug, Clone, Copy)] +pub(crate) struct TextContentBodyOptions { + detection: ContentCodingDetection, + max_decoded_bytes: Option, + deadline: Option, + retry_owner: TextContentRetryOwner, +} -async fn build_decoded_stream_reader(response: reqwest::Response) -> Result { - let headers = response.headers(); - let header_value = headers.get(CONTENT_ENCODING); - let mut encoding = header_value.and_then(|h| h.to_str().ok()).map(ToString::to_string); +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum TextContentRetryOwner { + #[default] + RequestStack, + DecodedBodyConsumer, +} - let stream_reader = StreamReader::new(response.bytes_stream().map_err(std::io::Error::other)); - let mut buf_reader = async_file_reader(stream_reader); +impl Default for TextContentBodyOptions { + fn default() -> Self { + Self { + detection: ContentCodingDetection::DeclaredOrLegacyTextMagic, + max_decoded_bytes: None, + deadline: None, + retry_owner: TextContentRetryOwner::RequestStack, + } + } +} - let peek = buf_reader.fill_buf().await?; - - if peek.len() >= 2 { - if is_gzip(&peek[0..2]) { - encoding = Some(ENCODING_GZIP.to_string()); - } else if is_deflate(&peek[0..2]) { - encoding = Some(ENCODING_DEFLATE.to_string()); +impl TextContentBodyOptions { + /// Selects the narrowly scoped HLS-manifest fallback detection and decoded-size limit. + pub(crate) fn hls_manifest(max_decoded_bytes: usize, deadline: Duration) -> Self { + Self { + detection: ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, + max_decoded_bytes: Some(max_decoded_bytes), + deadline: Some(deadline.max(Duration::from_millis(1))), + retry_owner: TextContentRetryOwner::DecodedBodyConsumer, } } - let reader: DynReader = if encoding.as_ref().is_some_and(|e| e.eq_ignore_ascii_case(ENCODING_GZIP)) { - Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader)) - } else if encoding.as_ref().is_some_and(|e| e.eq_ignore_ascii_case(ENCODING_DEFLATE)) { - Box::pin(async_compression::tokio::bufread::ZlibDecoder::new(buf_reader)) - } else { - Box::pin(buf_reader) + fn legacy_text_with_deadline(deadline: Option) -> Self { + Self { deadline: deadline.map(|value| value.max(Duration::from_millis(1))), ..Self::default() } + } +} + +/// Groups request-boundary and decoded-text-consumer options for one text fetch. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct TextContentFetchOptions { + request: RequestFetchOptions, + body: TextContentBodyOptions, +} + +impl TextContentFetchOptions { + pub(crate) const fn new(request: RequestFetchOptions, body: TextContentBodyOptions) -> Self { + Self { request, body } + } + + fn with_request_options(request: RequestFetchOptions) -> Self { + Self { body: TextContentBodyOptions::legacy_text_with_deadline(request.attempt_idle_timeout), request } + } +} + +async fn build_decoded_stream_reader(response: reqwest::Response) -> Result { + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrLegacyTextMagic) + .await + .map(|decoded| decoded.body) + .map_err(content_coding_error_to_io) +} + +fn content_coding_error_to_io(error: ContentCodingError) -> Error { + let kind = match &error { + ContentCodingError::PrefixRead(source) => source.kind(), + ContentCodingError::InvalidHeader + | ContentCodingError::Unsupported(_) + | ContentCodingError::EncodedPartialContent => ErrorKind::Other, + }; + Error::new(kind, error) +} + +fn content_body_read_error_to_io(error: ContentBodyReadError) -> Error { + match error { + ContentBodyReadError::Io(error) => error, + error @ ContentBodyReadError::LimitExceeded { .. } => Error::other(error), + error @ ContentBodyReadError::InvalidUtf8 { .. } => Error::new(ErrorKind::InvalidData, error), + } +} + +/// Builds a fixed/numeric text-response diagnostic without origin-controlled content. +pub(crate) fn text_response_error_log_label(error: &Error) -> String { + if let Some(error) = content_decoding_error_from_io(error) { + return format!("content_decoding coding={}", error.coding.as_http_token()); + } + if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { + return match error { + ContentBodyReadError::LimitExceeded { limit } => format!("decoded_body_limit limit={limit}"), + ContentBodyReadError::InvalidUtf8 { valid_up_to, error_len } => { + format!("invalid_utf8 valid_up_to={valid_up_to} error_len={error_len:?}") + } + ContentBodyReadError::Io(error) => format!("io kind={:?}", error.kind()), + }; + } + if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { + return match error { + ContentCodingError::InvalidHeader => "content_coding class=invalid_header".to_string(), + ContentCodingError::Unsupported(_) => "content_coding class=unsupported".to_string(), + ContentCodingError::EncodedPartialContent => "content_coding class=encoded_partial_content".to_string(), + ContentCodingError::PrefixRead(_) => "content_coding class=prefix_read".to_string(), + }; + } + if error.kind() == ErrorKind::TimedOut { + return "timeout".to_string(); + } + if is_http_body_transport_error(error) { + return "transport".to_string(); + } + format!("io kind={:?}", error.kind()) +} + +async fn read_text_response_with_body_options( + response: reqwest::Response, + body_options: TextContentBodyOptions, +) -> Result<(String, String, HeaderMap), Error> { + let request_url = response.url().to_string(); + let read = async move { + let mut decoded = + decode_response_to_identity(response, body_options.detection).await.map_err(content_coding_error_to_io)?; + if let (ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, Some(observation)) = + (body_options.detection, decoded.content_coding_observation()) + { + log_hls_origin_content_coding( + observation, + HlsOriginContentCodingObjectKind::Manifest, + false, + HlsOriginContentCodingSource::Legacy, + ); + } + let content = if let Some(max_decoded_bytes) = body_options.max_decoded_bytes { + read_utf8_limited(&mut decoded.body, max_decoded_bytes).await.map_err(content_body_read_error_to_io)? + } else { + let mut content = String::new(); + decoded.body.read_to_string(&mut content).await.map_err(|error| Error::new(error.kind(), error))?; + content + }; + Ok((content, decoded.final_url.to_string(), decoded.headers)) }; - Ok(reader) + if let Some(deadline) = body_options.deadline { + tokio::time::timeout(deadline, read).await.map_err(|_| { + Error::new( + ErrorKind::TimedOut, + format!("Timed out reading content body: {}", sanitize_sensitive_info(&request_url)), + ) + })? + } else { + read.await + } } #[allow(clippy::implicit_hasher)] @@ -1909,58 +2137,59 @@ async fn get_remote_content_as_stream_with_options( Ok((reader, response_url)) } -async fn read_stream_to_string_with_options( - stream: &mut DynReader, - response_url: &str, - options: RequestFetchOptions, -) -> Result { - let mut content = String::new(); - if let Some(timeout) = options.attempt_idle_timeout { - tokio::time::timeout(timeout, stream.read_to_string(&mut content)) - .await - .map_err(|_| { - Error::new( - ErrorKind::TimedOut, - format!( - "Timed out reading content body: {}", - sanitize_sensitive_info(response_url) - ), - ) - })? - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - } else { - stream - .read_to_string(&mut content) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - } - Ok(content) -} - -fn text_body_retry_values(app_config: &Arc, options: RequestFetchOptions) -> (u32, u64, f64) { - if options.attempt_idle_timeout.is_none() { +fn text_body_retry_values(app_config: &Arc, body_options: TextContentBodyOptions) -> (u32, u64, f64) { + if body_options.retry_owner != TextContentRetryOwner::DecodedBodyConsumer { return (1, 0, 1.0); } let config = app_config.config.load(); - let values = config.reverse_proxy.as_ref().map_or_else( - ResourceRetryConfig::get_default_retry_values, - |rp| rp.resource_retry.get_retry_values(), - ); + let values = config + .reverse_proxy + .as_ref() + .map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values()); drop(config); values } +fn should_retry_text_body_error(error: &Error) -> bool { + matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::ConnectionRefused) + || content_decoding_error_from_io(error).is_some() + || is_http_body_transport_error(error) + || error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|error| matches!(error, ContentCodingError::PrefixRead(_))) +} + async fn sleep_before_text_body_retry(attempt: u32, backoff_ms: u64, backoff_multiplier: f64, err: &Error) { let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); warn!( - "Text response body timed out, retrying in {}ms (attempt {}): {}", + "Text response body failed retryably, retrying in {}ms (attempt {}): {}", delay, attempt + 1, - sanitize_sensitive_info(err.to_string().as_str()) + text_response_error_log_label(err) ); tokio::time::sleep(Duration::from_millis(delay)).await; } +fn is_retryable_text_response_status(status: StatusCode) -> bool { + status.is_server_error() + || matches!( + status, + StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_EARLY + | StatusCode::TOO_MANY_REQUESTS + ) +} + +fn text_response_status_error(status: StatusCode, url: &Url) -> Error { + string_to_io_error(format!( + "Request failed ({}): {}", + format_http_status(status), + sanitize_sensitive_info(url.as_str()) + )) +} + async fn get_remote_content_with_options( app_config: &Arc, client: &reqwest::Client, @@ -1969,71 +2198,66 @@ async fn get_remote_content_with_options( url: &Url, options: RequestFetchOptions, ) -> Result<(String, String), Error> { - let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options); - for attempt in 0..max_attempts { - let (mut stream, response_url) = get_remote_content_as_stream_with_options(app_config, client, input, headers, url, options) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - match read_stream_to_string_with_options(&mut stream, response_url.as_str(), options).await { - Ok(content) => return Ok((content, response_url)), - Err(err) if err.kind() == ErrorKind::TimedOut && attempt + 1 < max_attempts => { - sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &err).await; - } - Err(err) => return Err(err), - } - } - Err(string_to_io_error("Text response body retry attempts exhausted")) + get_remote_content_with_headers_and_options( + app_config, + client, + input, + headers, + url, + TextContentFetchOptions::with_request_options(options), + ) + .await + .map(|(content, response_url, _)| (content, response_url)) } -async fn get_remote_content_with_headers( +async fn get_remote_content_with_headers_and_options( app_config: &Arc, client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, + options: TextContentFetchOptions, ) -> Result<(String, String, HeaderMap), Error> { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); + let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); + let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let merged = get_request_headers( - Some(&input.headers), - custom_headers.as_ref(), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - - let headers: HashMap = merged - .iter() - .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) - .collect(); - - let response = send_with_retry_and_provider(app_config, url, input.get_provider(), false, |resolved_url| { - get_client_request( + // This consumer owns the logical attempt budget whenever decoded-body retries are enabled. Provider failover + // and redirect hops remain bounded subrequests, but the configured retry count is not applied again below it. + for attempt in 0..max_attempts { + let response = match send_input_with_retry_and_provider_policy_with_options_result( + app_config, client, - input.method, - Some(&headers), - resolved_url, - None, - None, - default_user_agent.as_deref(), + input, + headers, + url, + attempt_options, ) - }) - .await?; - - let response_url = response.url().to_string(); - let response_headers = response.headers().clone(); - let mut stream = build_decoded_stream_reader(response).await?; - let mut content = String::new(); - stream - .read_to_string(&mut content) .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - Ok((content, response_url, response_headers)) + { + Ok(result) => result.response, + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + continue; + } + Err(error) => return Err(error), + }; + let status = response.status(); + if !status.is_success() { + if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue; + } + return Err(text_response_status_error(status, url)); + } + match read_text_response_with_body_options(response, options.body).await { + Ok(result) => return Ok(result), + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + } + Err(error) => return Err(error), + } + } + Err(string_to_io_error("Text response body retry attempts exhausted")) } #[allow(clippy::too_many_lines)] @@ -2046,126 +2270,68 @@ async fn get_remote_content_with_manual_redirects_and_options( max_redirects: usize, options: RequestFetchOptions, ) -> Result<(String, String), Error> { - let (max_body_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options); - - for body_attempt in 0..max_body_attempts { - let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - app_config, - client, - input, - headers, - url, - max_redirects, - options, - ) - .await? - .response; - let response_url = response.url().to_string(); - let mut stream = build_decoded_stream_reader(response).await?; - match read_stream_to_string_with_options(&mut stream, response_url.as_str(), options).await { - Ok(content) => return Ok((content, response_url)), - Err(err) if err.kind() == ErrorKind::TimedOut && body_attempt + 1 < max_body_attempts => { - sleep_before_text_body_retry(body_attempt, backoff_ms, backoff_multiplier, &err).await; - } - Err(err) => return Err(err), - } - } - Err(string_to_io_error("Text response body retry attempts exhausted")) + get_remote_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + url, + max_redirects, + TextContentFetchOptions::with_request_options(options), + ) + .await + .map(|(content, response_url, _)| (content, response_url)) } -async fn get_remote_content_with_manual_redirects_and_headers( +async fn get_remote_content_with_manual_redirects_and_headers_and_options( app_config: &Arc, client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, max_redirects: usize, + options: TextContentFetchOptions, ) -> Result<(String, String, HeaderMap), Error> { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); + let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); + let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let merged = get_request_headers( - Some(&input.headers), - custom_headers.as_ref(), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - - let headers: HashMap = merged - .iter() - .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) - .collect(); - - let mut current_url = url.clone(); - let mut current_headers = headers; - let mut remaining_redirects = max_redirects; - loop { - let response = - send_with_retry_and_provider(app_config, ¤t_url, input.get_provider(), true, |resolved_url| { - get_client_request( - client, - input.method, - Some(¤t_headers), - resolved_url, - None, - None, - default_user_agent.as_deref(), - ) - }) - .await?; - let response_base_url = response.url().clone(); - - if response.status().is_redirection() { - if remaining_redirects == 0 { - return Err(string_to_io_error(format!( - "Too many redirects while requesting {}", - sanitize_sensitive_info(url.as_str()) - ))); + // Manual redirects retain their own credential-scrubbing loop inside each caller-owned logical attempt. + for attempt in 0..max_attempts { + let response = match send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config, + client, + input, + headers, + url, + max_redirects, + attempt_options, + ) + .await + { + Ok(result) => result.response, + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + continue; } - - let Some(location) = response.headers().get(reqwest::header::LOCATION) else { - return Err(string_to_io_error(format!( - "Redirect response missing location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let Ok(location_str) = location.to_str() else { - return Err(string_to_io_error(format!( - "Redirect response contains invalid location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let next_url = - response_base_url.join(location_str).or_else(|_| Url::parse(location_str)).map_err(|_| { - string_to_io_error(format!( - "Redirect response contains invalid location URL for {}", - sanitize_sensitive_info(current_url.as_str()) - )) - })?; - - if !same_origin(&response_base_url, &next_url) { - strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); + Err(error) => return Err(error), + }; + let status = response.status(); + if !status.is_success() { + if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue; } - current_url = next_url; - remaining_redirects = remaining_redirects.saturating_sub(1); - continue; + return Err(text_response_status_error(status, url)); + } + match read_text_response_with_body_options(response, options.body).await { + Ok(result) => return Ok(result), + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + } + Err(error) => return Err(error), } - - let response_url = response.url().to_string(); - let response_headers = response.headers().clone(); - let mut stream = build_decoded_stream_reader(response).await?; - let mut content = String::new(); - stream - .read_to_string(&mut content) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - return Ok((content, response_url, response_headers)); } + Err(string_to_io_error("Text response body retry attempts exhausted")) } fn same_origin(lhs: &Url, rhs: &Url) -> bool { @@ -2174,7 +2340,8 @@ fn same_origin(lhs: &Url, rhs: &Url) -> bool { && lhs.port_or_known_default() == rhs.port_or_known_default() } -fn is_safe_cross_origin_redirect_header(key: &str) -> bool { +/// Reports whether a request header may be retained when the target origin changes. +pub(crate) fn is_safe_cross_origin_redirect_header(key: &str) -> bool { key.eq_ignore_ascii_case("accept") || key.eq_ignore_ascii_case("accept-encoding") || key.eq_ignore_ascii_case("accept-language") @@ -2189,14 +2356,8 @@ fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HashMap Result<(tempfile::NamedTempFile, File), Error> { - let parent = file_path - .parent() - .filter(|path| !path.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - let temp_file = tempfile::Builder::new() - .prefix(".tuliprox-download-") - .suffix(".tmp") - .tempfile_in(parent)?; + let parent = file_path.parent().filter(|path| !path.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")); + let temp_file = tempfile::Builder::new().prefix(".tuliprox-download-").suffix(".tmp").tempfile_in(parent)?; let output_file = File::from_std(temp_file.reopen()?); Ok((temp_file, output_file)) } @@ -2224,14 +2385,11 @@ async fn copy_local_epg_file_to_persist( if read == 0 { break; } - copied = copied.checked_add(read as u64).ok_or_else(|| { - string_to_io_error(format!("Local EPG file size overflow for {}", file_path.display())) - })?; + copied = copied + .checked_add(read as u64) + .ok_or_else(|| string_to_io_error(format!("Local EPG file size overflow for {}", file_path.display())))?; if max_bytes.is_some_and(|max| copied > max) { - return Err(string_to_io_error(format!( - "Local EPG file {} exceeds configured limit", - file_path.display() - ))); + return Err(string_to_io_error(format!("Local EPG file {} exceeds configured limit", file_path.display()))); } writer.write_all(&buffer[..read]).await?; } @@ -2257,10 +2415,7 @@ async fn download_epg_content_as_file( match url.scheme() { "file" => { let file_path = url.to_file_path().map_err(|()| { - Error::new( - ErrorKind::Unsupported, - format!("Unknown file {}", sanitize_sensitive_info(url_str)), - ) + Error::new(ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str))) })?; if file_path.exists() { copy_local_epg_file_to_persist(&file_path, persist_filepath, max_bytes).await @@ -2276,10 +2431,7 @@ async fn download_epg_content_as_file( headers, &url, persist_filepath, - FileDownloadOptions { - max_bytes, - atomic_write: true, - }, + FileDownloadOptions { max_bytes, atomic_write: true }, ) .await } @@ -2301,7 +2453,7 @@ pub async fn download_text_content( persist_filepath: Option, trace_log: bool, ) -> Result<(String, String), Error> { - download_text_content_with_options( + Box::pin(download_text_content_with_options( app_config, client, input, @@ -2309,7 +2461,7 @@ pub async fn download_text_content( persist_filepath, trace_log, RequestFetchOptions::default(), - ) + )) .await } @@ -2366,16 +2518,37 @@ pub async fn download_text_content_with_headers( input: &InputSource, headers: Option<&HeaderMap>, trace_log: bool, +) -> Result<(String, String, HeaderMap), Error> { + Box::pin(download_text_content_with_headers_and_options( + app_config, + client, + input, + headers, + trace_log, + TextContentFetchOptions::default(), + )) + .await +} + +pub(crate) async fn download_text_content_with_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, + options: TextContentFetchOptions, ) -> Result<(String, String, HeaderMap), Error> { let start_time = tokio::time::Instant::now(); let result = if let Ok(url) = input.url.parse::() { let result = if url.scheme() == "file" { match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())), + Ok(file_path) => { + get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) + } Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), } } else { - get_remote_content_with_headers(app_config, client, input, headers, &url).await + get_remote_content_with_headers_and_options(app_config, client, input, headers, &url, options).await }; result } else { @@ -2406,7 +2579,7 @@ pub async fn download_text_content_with_manual_redirects( trace_log: bool, max_redirects: usize, ) -> Result<(String, String), Error> { - download_text_content_with_manual_redirects_and_options( + Box::pin(download_text_content_with_manual_redirects_and_options( app_config, client, input, @@ -2415,7 +2588,7 @@ pub async fn download_text_content_with_manual_redirects( trace_log, max_redirects, RequestFetchOptions::default(), - ) + )) .await } @@ -2484,16 +2657,48 @@ pub async fn download_text_content_with_manual_redirects_and_headers( headers: Option<&HeaderMap>, trace_log: bool, max_redirects: usize, +) -> Result<(String, String, HeaderMap), Error> { + Box::pin(download_text_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + trace_log, + max_redirects, + TextContentFetchOptions::default(), + )) + .await +} + +pub(crate) async fn download_text_content_with_manual_redirects_and_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, + max_redirects: usize, + options: TextContentFetchOptions, ) -> Result<(String, String, HeaderMap), Error> { let start_time = tokio::time::Instant::now(); let result = if let Ok(url) = input.url.parse::() { let result = if url.scheme() == "file" { match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())), + Ok(file_path) => { + get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) + } Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), } } else { - get_remote_content_with_manual_redirects_and_headers(app_config, client, input, headers, &url, max_redirects).await + get_remote_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + &url, + max_redirects, + options, + ) + .await }; result } else { @@ -2540,7 +2745,7 @@ pub async fn download_text_content_as_stream( debug!("Persisted {size} bytes"); })), ) - .await; + .await; Ok((tee_reader, response_url)) } else { Ok((content, response_url)) @@ -2737,33 +2942,46 @@ pub fn should_trigger_failover(status: StatusCode) -> bool { #[cfg(test)] mod tests { use super::{ - is_safe_cross_origin_redirect_header, - get_input_epg_content_as_file, next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, - resolve_attempt_target, same_origin, send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, - send_with_retry_and_provider, send_with_retry_and_provider_policy, should_try_next_ip_on_connect_error, - strip_sensitive_headers_for_cross_origin_redirect, InputEpgFileRequest, RequestFetchOptions, + download_text_content, download_text_content_with_headers_and_options, get_input_epg_content_as_file, + get_remote_content_as_stream, is_safe_cross_origin_redirect_header, next_provider_url_index, + preview_request_diagnostics_for_logging, preview_request_target_for_logging, resolve_attempt_target, + same_origin, send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_input_with_retry_and_provider_policy_with_options_result, send_with_retry_and_provider, + send_with_retry_and_provider_policy, should_retry_text_body_error, should_try_next_ip_on_connect_error, + strip_sensitive_headers_for_cross_origin_redirect, text_response_error_log_label, InputEpgFileRequest, + RequestFetchOptions, TextContentBodyOptions, TextContentFetchOptions, }; use crate::{ model::{ AppConfig, Config, ConfigInput, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, ReverseProxyConfig, SourcesConfig, }, - utils::{FileLockManager}, + utils::{ + content_coding::{ContentCoding, ContentCodingError, ContentDecodingIoError, OutboundContentCodingPolicy}, + FileLockManager, + }, }; use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ - ConfigPaths, ConfigProviderDto, DnsScheme, InputFetchMethod, OnConnectErrorPolicy, ProviderDnsDto, - ProviderUrlSelectionPolicy, + use flate2::{ + write::{GzEncoder, ZlibEncoder}, + Compression, + }; + use reqwest::header::{HeaderMap, HeaderValue, ACCEPT_ENCODING, COOKIE}; + use shared::{ + defaults::DEFAULT_USER_AGENT, + model::{ + ConfigPaths, ConfigProviderDto, DnsScheme, InputFetchMethod, OnConnectErrorPolicy, ProviderDnsDto, + ProviderUrlSelectionPolicy, + }, + utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}, }; - use shared::utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}; - use shared::defaults::{DEFAULT_USER_AGENT}; - use std::{ collections::{HashMap, HashSet}, + io::{Error, ErrorKind, Write}, net::SocketAddr, path::{Path, PathBuf}, sync::{ - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, }, time::Duration, @@ -2771,6 +2989,7 @@ mod tests { use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, + sync::{oneshot, Mutex}, }; use url::Url; @@ -2839,11 +3058,12 @@ mod tests { assert!(atomic_download_temp_files(dir.path()).await.is_empty()); } - fn make_provider_with_dns(keep_vhost: bool, on_connect_error: OnConnectErrorPolicy, ips: Vec<&str>) -> Arc { - let parsed_ips = ips - .into_iter() - .map(|raw| raw.parse().expect("ip must parse")) - .collect::>(); + fn make_provider_with_dns( + keep_vhost: bool, + on_connect_error: OnConnectErrorPolicy, + ips: Vec<&str>, + ) -> Arc { + let parsed_ips = ips.into_iter().map(|raw| raw.parse().expect("ip must parse")).collect::>(); let dto = ConfigProviderDto { name: "provider-a".into(), urls: vec!["http://example.com".into()], @@ -3007,9 +3227,7 @@ mod tests { let dir = tempfile::tempdir().expect("temp dir"); let source = dir.path().join("source.ics"); let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") - .await - .expect("write source"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); tokio::fs::write(&persist, b"old cache").await.expect("write old cache"); let source_url = Url::from_file_path(&source).expect("file url"); let app_config = make_test_app_config(Config::default()); @@ -3076,9 +3294,7 @@ mod tests { let dir = tempfile::tempdir().expect("temp dir"); let source = dir.path().join("source.ics"); let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") - .await - .expect("write source"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); tokio::fs::create_dir(&persist).await.expect("create conflicting destination"); let app_config = make_test_app_config(Config::default()); let client = make_epg_test_client(); @@ -3167,9 +3383,7 @@ mod tests { let dir = tempfile::tempdir().expect("temp dir"); let source = dir.path().join("source.ics"); let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") - .await - .expect("write source"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); let app_config = make_test_app_config(Config::default()); let client = make_epg_test_client(); let input = ConfigInput::default(); @@ -3324,10 +3538,7 @@ mod tests { assert_eq!(second_result.expect("second download"), persist); assert_eq!(accepted.load(Ordering::SeqCst), 2); assert_eq!(max_active.load(Ordering::SeqCst), 1); - assert_eq!( - tokio::fs::read(&persist).await.expect("read refreshed cache"), - b"BEGIN:VCALENDAR\nEND:VCALENDAR\n" - ); + assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); assert!(atomic_download_temp_files(dir.path()).await.is_empty()); server_handle.abort(); } @@ -3409,7 +3620,8 @@ mod tests { #[test] fn test_try_next_ip_policy_uses_next_ip_until_exhausted() { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let provider = + make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); let url = Url::parse("http://example.com/live").expect("url parse should work"); let mut tried = HashSet::new(); @@ -3422,7 +3634,8 @@ mod tests { #[test] fn test_preview_request_target_for_logging_does_not_advance_dns_rotation() { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let provider = + make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); let url = Url::parse("http://example.com/live").expect("url parse should work"); let preview = preview_request_target_for_logging(&url, Some(&provider)); @@ -3433,7 +3646,8 @@ mod tests { assert_eq!(first.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); assert_eq!(second.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.2".to_string())); - let provider_https = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let provider_https = + make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); let https_url = Url::parse("https://example.com/live").expect("url parse should work"); let https_preview = preview_request_target_for_logging(&https_url, Some(&provider_https)); @@ -3449,10 +3663,7 @@ mod tests { fn test_preview_request_target_for_logging_uses_preferred_provider_index() { let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { name: "provider-a".into(), - urls: vec![ - "http://provider-a.example".into(), - "http://provider-b.example".into(), - ], + urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), dns: None, })); @@ -3483,9 +3694,8 @@ mod tests { tokio::spawn(async move { let mut buf = vec![0_u8; 2048]; let _ = socket.read(&mut buf).await; - let response_head = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n" - ); + let response_head = + format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); let _ = socket.write_all(response_head.as_bytes()).await; let _ = socket.write_all(body).await; let _ = socket.shutdown().await; @@ -3498,12 +3708,7 @@ mod tests { async fn start_delayed_http_server_with_body( body: &'static [u8], - ) -> std::io::Result<( - SocketAddr, - Arc, - Arc, - tokio::task::JoinHandle<()>, - )> { + ) -> std::io::Result<(SocketAddr, Arc, Arc, tokio::task::JoinHandle<()>)> { let listener = TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; let accepted = Arc::new(AtomicUsize::new(0)); @@ -3528,9 +3733,8 @@ mod tests { let mut buf = vec![0_u8; 2048]; let _ = socket.read(&mut buf).await; tokio::time::sleep(Duration::from_millis(100)).await; - let response_head = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n" - ); + let response_head = + format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); let _ = socket.write_all(response_head.as_bytes()).await; let _ = socket.write_all(body).await; let _ = socket.shutdown().await; @@ -3573,6 +3777,596 @@ mod tests { start_plain_http_server_with_body(b"ok").await } + async fn start_recording_http_server( + responses: Vec, + ) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { + start_recording_http_byte_server(responses.into_iter().map(String::into_bytes).collect()).await + } + + async fn start_recording_http_byte_server( + responses: Vec>, + ) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let responses = Arc::new(responses); + let response_index = Arc::new(AtomicUsize::new(0)); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let responses = Arc::clone(&responses); + let response_index = Arc::clone(&response_index); + tokio::spawn(async move { + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") || request.len() >= 16 * 1024 { + break; + } + } + requests.lock().await.push(String::from_utf8_lossy(&request).into_owned()); + let index = response_index.fetch_add(1, Ordering::SeqCst); + let Some(response) = responses.get(index).or_else(|| responses.last()) else { + return; + }; + let _ = socket.write_all(response).await; + let _ = socket.shutdown().await; + }); + } + }); + + Ok((addr, requests, handle)) + } + + async fn start_hanging_http_server( + ) -> std::io::Result<(SocketAddr, oneshot::Receiver<()>, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let (request_seen_tx, request_seen_rx) = oneshot::channel(); + let handle = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + let _ = request_seen_tx.send(()); + std::future::pending::<()>().await; + drop(socket); + }); + + Ok((addr, request_seen_rx, handle)) + } + + fn response_with_body(status: &str, body: &str) -> String { + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()) + } + + fn response_with_byte_body(status: &str, body: &[u8]) -> Vec { + let mut response = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).into_bytes(); + response.extend_from_slice(body); + response + } + + fn gzip_encoded(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("gzip input"); + encoder.finish().expect("gzip output") + } + + fn zlib_encoded(body: &[u8]) -> Vec { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("zlib input"); + encoder.finish().expect("zlib output") + } + + fn request_header_value<'a>(request: &'a str, expected_name: &str) -> Option<&'a str> { + request.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) + }) + } + + fn test_retry_config(max_attempts: u32) -> Arc { + let mut config = Config { connect_timeout_secs: 1, ..Config::default() }; + config.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { + max_attempts, + backoff_millis: 1, + backoff_multiplier: 1.0, + ..ResourceRetryConfig::default() + }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + make_test_app_config(config) + } + + fn identity_fetch_options() -> RequestFetchOptions { + RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)) + .with_content_coding(OutboundContentCodingPolicy::Identity) + } + + #[test] + fn physical_request_attempt_applies_target_and_final_content_coding_options() { + let client = reqwest::Client::builder().no_proxy().build().expect("test client"); + let request_url = Url::parse("http://origin.example/manifest.m3u8").expect("request URL"); + let effective_url = Url::parse("http://127.0.0.1/manifest.m3u8").expect("effective URL"); + let target = super::AttemptTarget { + request_url: request_url.clone(), + effective_url: effective_url.clone(), + host_header: Some("origin.example".to_string()), + sni_host: None, + connect_ip: Some("127.0.0.1".parse().expect("connect IP")), + dns_host: Some("origin.example".to_string()), + }; + let builder = + client.get(request_url).header(ACCEPT_ENCODING, "gzip").header(reqwest::header::HOST, "wrong.example"); + + let (_, request) = super::prepare_physical_request_attempt(builder, &target, identity_fetch_options()) + .expect("physical request should build"); + + assert_eq!(request.url(), &effective_url); + assert_eq!(request.headers()[reqwest::header::HOST], "origin.example"); + assert_eq!(request.headers()[ACCEPT_ENCODING], "identity"); + assert_eq!(request.timeout(), Some(&Duration::from_secs(1))); + } + + #[tokio::test(start_paused = true)] + async fn provider_failover_only_default_options_keep_default_idle_timeout() { + let (addr, request_seen, server) = start_hanging_http_server().await.expect("hanging origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let app_config = test_retry_config(5); + let client = reqwest::Client::builder().no_proxy().build().expect("test client"); + let request_url = url.clone(); + let request = tokio::spawn(async move { + send_input_with_retry_and_provider_policy_with_options_result( + &app_config, + &client, + &input, + None, + &request_url, + RequestFetchOptions::default().without_resource_retries(), + ) + .await + }); + + // A ready task prevents Tokio's paused clock from auto-advancing through the default timeout while the local + // TCP handshake is still in progress. After the request arrives, this guard stops and the test advances time + // explicitly to the production deadline. + let hold_virtual_time = Arc::new(AtomicBool::new(true)); + let hold_virtual_time_for_task = Arc::clone(&hold_virtual_time); + let virtual_time_guard = tokio::spawn(async move { + while hold_virtual_time_for_task.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }); + let request_seen = request_seen.await; + hold_virtual_time.store(false, Ordering::SeqCst); + virtual_time_guard.await.expect("virtual time guard should stop"); + request_seen.expect("origin should receive the request"); + tokio::time::advance(Duration::from_secs(crate::api::model::STREAM_IDLE_TIMEOUT + 1)).await; + tokio::task::yield_now().await; + if !request.is_finished() { + request.abort(); + panic!("provider-only request lost the default idle-timeout guard"); + } + + let error = match request.await.expect("request task should join") { + Ok(_) => panic!("hanging request must time out"), + Err(error) => error, + }; + assert_eq!(error.kind(), ErrorKind::TimedOut); + server.abort(); + } + + #[test] + fn content_coding_prefix_read_error_is_retryable_but_unsupported_coding_is_not() { + let prefix_read = + Error::other(ContentCodingError::PrefixRead(Error::new(ErrorKind::UnexpectedEof, "prefix truncated"))); + let unsupported = Error::other(ContentCodingError::Unsupported("compress".to_string())); + + assert!(should_retry_text_body_error(&prefix_read)); + assert!(!should_retry_text_body_error(&unsupported)); + } + + #[test] + fn text_response_error_log_labels_never_expose_origin_controlled_details() { + let unsupported = Error::other(ContentCodingError::Unsupported("signed-token-secret".to_string())); + assert_eq!(text_response_error_log_label(&unsupported), "content_coding class=unsupported"); + assert!(!text_response_error_log_label(&unsupported).contains("signed-token-secret")); + + let decoding = Error::new(ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Zstd }); + assert_eq!(text_response_error_log_label(&decoding), "content_decoding coding=zstd"); + } + + fn test_input_source(url: String, provider: Option>) -> InputSource { + InputSource { + name: Arc::from("test"), + url, + provider, + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::new(), + } + } + + #[tokio::test] + async fn text_content_download_decodes_headerless_gzip_and_zlib() { + const TEXT: &[u8] = b"headerless legacy text\n"; + + for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { + let (addr, requests, handle) = + start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (content, _) = + download_text_content(&test_retry_config(1), &make_epg_test_client(), &input, None, None, false) + .await + .unwrap_or_else(|error| panic!("headerless {label} text should decode: {error}")); + + assert_eq!(content.as_bytes(), TEXT, "failed for {label}"); + assert_eq!(requests.lock().await.len(), 1); + handle.abort(); + } + } + + #[tokio::test] + async fn text_content_stream_decodes_headerless_gzip_and_zlib() { + const TEXT: &[u8] = b"streamed legacy text\n"; + + for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { + let (addr, requests, handle) = + start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (mut reader, _) = + get_remote_content_as_stream(&test_retry_config(1), &make_epg_test_client(), &input, None, &url) + .await + .unwrap_or_else(|error| panic!("headerless {label} stream should decode: {error}")); + let mut content = Vec::new(); + reader.read_to_end(&mut content).await.expect("read decoded stream"); + + assert_eq!(content, TEXT, "failed for {label}"); + assert_eq!(requests.lock().await.len(), 1); + handle.abort(); + } + } + + #[tokio::test] + async fn content_coding_identity_wins_after_input_and_client_header_merge() { + let (addr, requests, handle) = start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let mut input = test_input_source(url.to_string(), None); + input.headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); + let mut client_headers = HeaderMap::new(); + client_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("br")); + + let response = send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + Some(&client_headers), + &url, + identity_fetch_options(), + ) + .await + .expect("origin request should succeed"); + drop(response); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 1); + assert_eq!(request_header_value(&captured[0], "accept-encoding"), Some("identity")); + handle.abort(); + } + + #[tokio::test] + async fn content_coding_identity_is_reapplied_for_retry() { + let responses = vec![response_with_body("500 Internal Server Error", ""), response_with_body("200 OK", "ok")]; + let (addr, requests, handle) = + start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(2), + &make_epg_test_client(), + &input, + None, + &url, + identity_fetch_options(), + ) + .await + .expect("retry should reach successful response"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn content_coding_identity_manifest_retries_temporary_body_transport_error() { + let truncated = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\n#EXTM3U\n"; + let responses = vec![truncated.to_string(), response_with_body("200 OK", "#EXTM3U\nsegment.ts\n")]; + let (addr, requests, handle) = + start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (manifest, _, _) = download_text_content_with_headers_and_options( + &test_retry_config(2), + &make_epg_test_client(), + &input, + None, + false, + TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ), + ) + .await + .expect("temporary body transport failure should retry"); + + assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn text_content_manifest_uses_one_budget_for_status_decoder_and_success() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let responses = vec![ + response_with_body("503 Service Unavailable", ""), + corrupt_gzip.to_string(), + response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), + ]; + let (addr, requests, handle) = + start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + let (manifest, _, _) = download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect("third logical attempt should succeed"); + + assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); + let captured = requests.lock().await; + assert_eq!(captured.len(), 3); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn text_content_manifest_retry_budget_is_not_multiplied_by_inner_status_retries() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let responses = vec![ + response_with_body("503 Service Unavailable", ""), + response_with_body("502 Bad Gateway", ""), + corrupt_gzip.to_string(), + response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), + ]; + let (addr, requests, handle) = + start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect_err("three logical failures must exhaust the configured budget"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 3, "the fourth success response must not be requested"); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn text_content_manifest_repeated_decoder_failures_stop_at_configured_budget() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let (addr, requests, handle) = + start_recording_http_server(vec![corrupt_gzip.to_string()]).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect_err("decoder failures must exhaust the configured budget"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 3); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn content_coding_identity_is_reapplied_after_provider_url_switch() { + let (first_addr, first_requests, first_handle) = + start_recording_http_server(vec![response_with_body("502 Bad Gateway", "")]) + .await + .expect("first provider origin should start"); + let (second_addr, second_requests, second_handle) = + start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("second provider origin should start"); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![format!("http://{first_addr}").into(), format!("http://{second_addr}").into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let url = Url::parse("provider://provider-a/manifest.m3u8").expect("provider URL"); + let input = test_input_source(url.to_string(), Some(provider)); + + send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + None, + &url, + identity_fetch_options(), + ) + .await + .expect("provider failover should reach successful response"); + + let first = first_requests.lock().await; + let second = second_requests.lock().await; + assert_eq!(first.len(), 1); + assert_eq!(second.len(), 1); + assert_eq!(request_header_value(&first[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&second[0], "accept-encoding"), Some("identity")); + first_handle.abort(); + second_handle.abort(); + } + + #[tokio::test] + async fn content_coding_identity_is_reapplied_for_same_origin_manual_redirect() { + let redirect = "HTTP/1.1 302 Found\r\nLocation: /final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (addr, requests, handle) = + start_recording_http_server(vec![redirect.to_string(), response_with_body("200 OK", "ok")]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/entry.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + None, + &url, + 2, + identity_fetch_options(), + ) + .await + .expect("same-origin redirect should succeed"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); + } + + #[tokio::test] + async fn content_coding_identity_survives_cross_origin_redirect_credential_scrubbing() { + let (target_addr, target_requests, target_handle) = + start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("redirect target should start"); + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + let (entry_addr, entry_requests, entry_handle) = + start_recording_http_server(vec![redirect]).await.expect("redirect entry should start"); + let url = Url::parse(&format!("http://{entry_addr}/entry.m3u8")).expect("entry URL"); + let mut input = test_input_source(url.to_string(), None); + input.headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); + let mut client_headers = HeaderMap::new(); + client_headers.insert(COOKIE, HeaderValue::from_static("sid=client-secret")); + + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + Some(&client_headers), + &url, + 2, + identity_fetch_options(), + ) + .await + .expect("cross-origin redirect should succeed"); + + let entry = entry_requests.lock().await; + assert_eq!(entry.len(), 1); + assert_eq!(request_header_value(&entry[0], "authorization"), Some("Bearer input-secret")); + assert_eq!(request_header_value(&entry[0], "cookie"), Some("sid=client-secret")); + drop(entry); + let target = target_requests.lock().await; + assert_eq!(target.len(), 1); + assert_eq!(request_header_value(&target[0], "accept-encoding"), Some("identity")); + assert!(request_header_value(&target[0], "authorization").is_none()); + assert!(request_header_value(&target[0], "cookie").is_none()); + entry_handle.abort(); + target_handle.abort(); + } + #[tokio::test] async fn manual_redirect_provider_failover_restarts_from_provider_entry() { let (redirect_addr, redirect_hits, redirect_handle) = match start_plain_http_server_with_response( @@ -3588,26 +4382,19 @@ mod tests { Err(err) => panic!("failed to start redirect target server: {err}"), }; let redirect_url = format!("http://127.0.0.1:{}/redirected", redirect_addr.port()); - let provider_a_response = format!( - "HTTP/1.1 302 Found\r\nLocation: {redirect_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - ); + let provider_a_response = + format!("HTTP/1.1 302 Found\r\nLocation: {redirect_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); let provider_entrypoint = start_plain_http_server_with_response(provider_a_response) .await .expect("provider a test server should start"); let successful_mirror = start_plain_http_server_with_body(b"provider-b").await.expect("provider b test server should start"); - let mut cfg = Config { - connect_timeout_secs: 1, - ..Config::default() - }; + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; cfg.reverse_proxy = Some(ReverseProxyConfig { resource_rewrite_disabled: false, rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts: 1, - ..ResourceRetryConfig::default() - }, + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, disabled_header: None, stream: None, cache: None, @@ -3675,26 +4462,18 @@ mod tests { let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { Ok(server) => server, Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!( - "skipping test_provider_request_chain_starts_from_last_successful_url: {err}" - ); + eprintln!("skipping test_provider_request_chain_starts_from_last_successful_url: {err}"); return; } Err(err) => panic!("failed to start test http server: {err}"), }; - let mut cfg = Config { - connect_timeout_secs: 1, - ..Config::default() - }; + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; cfg.accept_insecure_ssl_certificates = true; cfg.reverse_proxy = Some(ReverseProxyConfig { resource_rewrite_disabled: false, rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts: 1, - ..ResourceRetryConfig::default() - }, + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, disabled_header: None, stream: None, cache: None, @@ -3757,10 +4536,7 @@ mod tests { Err(err) => panic!("failed to start test http server: {err}"), }; - let mut cfg = Config { - connect_timeout_secs: 1, - ..Config::default() - }; + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; cfg.accept_insecure_ssl_certificates = true; cfg.reverse_proxy = Some(ReverseProxyConfig { resource_rewrite_disabled: false, @@ -3794,10 +4570,11 @@ mod tests { })); let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); - let result = send_with_retry_and_provider_policy(&app_config, &url, Some(&provider), false, false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await; + let result = + send_with_retry_and_provider_policy(&app_config, &url, Some(&provider), false, false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await; assert!(result.is_err(), "retry disabled must not fail over to the second provider URL"); assert_eq!(accepted_b.load(Ordering::SeqCst), 0, "fallback provider URL must not be contacted"); @@ -3819,18 +4596,12 @@ mod tests { Err(err) => panic!("failed to start test http server: {err}"), }; - let mut cfg = Config { - connect_timeout_secs: 1, - ..Config::default() - }; + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; cfg.accept_insecure_ssl_certificates = true; cfg.reverse_proxy = Some(ReverseProxyConfig { resource_rewrite_disabled: false, rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts: 1, - ..ResourceRetryConfig::default() - }, + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, disabled_header: None, stream: None, cache: None, @@ -3903,18 +4674,12 @@ mod tests { Err(err) => panic!("failed to start test http server: {err}"), }; - let mut cfg = Config { - connect_timeout_secs: 1, - ..Config::default() - }; + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; cfg.accept_insecure_ssl_certificates = true; cfg.reverse_proxy = Some(ReverseProxyConfig { resource_rewrite_disabled: false, rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts: 1, - ..ResourceRetryConfig::default() - }, + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, disabled_header: None, stream: None, cache: None, @@ -3935,9 +4700,10 @@ mod tests { let provider_rotate = make_provider_with_dns(false, OnConnectErrorPolicy::RotateProviderUrl, vec!["192.168.0.1", "127.0.0.1"]); - let result_rotate = send_with_retry_and_provider(&app_config, &url, Some(&provider_rotate), false, |resolved_url| { - client.get(resolved_url.clone()) - }) + let result_rotate = + send_with_retry_and_provider(&app_config, &url, Some(&provider_rotate), false, |resolved_url| { + client.get(resolved_url.clone()) + }) .await; assert!(result_rotate.is_err(), "without try_next_ip policy the request should fail"); @@ -3947,7 +4713,7 @@ mod tests { send_with_retry_and_provider(&app_config, &url, Some(&provider_try_next), false, |resolved_url| { client.get(resolved_url.clone()) }) - .await; + .await; assert!(result_try_next.is_ok(), "try_next_ip should succeed by trying the second IP"); assert_eq!(accepted.load(Ordering::SeqCst), 1, "server should be reached exactly once"); diff --git a/backend/src/utils/network/xtream.rs b/backend/src/utils/network/xtream.rs index c277fd1af..5a0ebc66c 100644 --- a/backend/src/utils/network/xtream.rs +++ b/backend/src/utils/network/xtream.rs @@ -618,6 +618,7 @@ mod tests { input_name: "provider".intern(), channel_no: 1, source_ordinal: 1, + input_stream_id: "813563".intern(), } } diff --git a/docs/src/configuration/hls-cache-state-machine.md b/docs/src/configuration/hls-cache-state-machine.md index 282b53590..d057b420e 100644 --- a/docs/src/configuration/hls-cache-state-machine.md +++ b/docs/src/configuration/hls-cache-state-machine.md @@ -23,12 +23,21 @@ For an operator-friendly introduction, see [Shared HLS Sessions](./shared-hls-se | Identifier | Scope | Meaning | | :--- | :--- | :--- | -| `HlsSessionKey` | Shared content | Stable tuple of `input_id`, HLS kind, and `stream_ref`. It does not include the origin URL, provider URL, username, or password. | +| `HlsSessionKey` | Shared content | Stable tuple of `input_id`, the literal HLS kind, and `stream_ref`, where `stream_ref` is the unchanged original `input_stream_id`. | | `proxy_session_id` | Shared public URL identity | Opaque token derived from `HlsSessionKey` and the configured secret. It identifies the shared content session in canonical URLs. | | `HlsPlaybackFamilyKey` | User/client family | Tuple of Tuliprox username and client fingerprint key. It groups playback attempts by user and client. | | `hls_access_lease_id` | Per playback URL identity | Random lookup key for one server-side `HlsAccessLease`. It is not a shared-content identity. | | HLS cache user session token | Per playback admission | Internal Tuliprox user-session token associated with an access lease. | +The canonical internal key is `input:|hls|`. The input stream ID is captured from the input +playlist item before target mapping: the provider/origin stream ID as a decimal string for Xtream, or the parser-resolved +string ID for M3U. The latter can be alphanumeric or a stable URL-derived hash. `target_id`, `virtual_id`, origin URLs, +credentials, and provider mirrors do not participate in session identity. Therefore, different targets reuse one +`HlsSession` when both `input_id` and `input_stream_id` match, while their access leases remain separate. + +The runtime assumes that `(input_id, input_stream_id)` uniquely identifies one input stream. A missing input stream ID is +not replaced with `virtual_id`; the Shared HLS entry fails instead of joining an ambiguously identified session. + The public canonical paths are: ```text diff --git a/docs/src/configuration/shared-hls-runtime-flow.md b/docs/src/configuration/shared-hls-runtime-flow.md index 091ea5dff..2356c73dd 100644 --- a/docs/src/configuration/shared-hls-runtime-flow.md +++ b/docs/src/configuration/shared-hls-runtime-flow.md @@ -9,13 +9,16 @@ reference, see [HLS Cache State Machines](./hls-cache-state-machine.md). | Identifier | Scope | Created by | Purpose | | :--- | :--- | :--- | :--- | -| `HlsSessionKey` | Shared content | Built from `input_id`, the literal HLS kind, and `stream_ref`. | Stable internal key for one shared live HLS session. It does not include provider username, password, or origin URL. | -| `proxy_session_id` | Public shared content URL | Derived from `HlsSessionKey` and the configured secret. | Opaque URL token for the shared session. The same input and stream reference produce the same public session ID while the secret stays stable. | +| `HlsSessionKey` | Shared content | Built as `input:\|hls\|`. | Stable internal key for one shared live HLS session. The input stream ID is captured before target mapping. | +| `proxy_session_id` | Public shared content URL | Derived from `HlsSessionKey` and the configured secret. | Opaque URL token for the shared session. The same input and original input stream ID produce the same public session ID while the secret stays stable. | | `HlsPlaybackFamilyKey` | Playback family | Built from Tuliprox username and client fingerprint. | Groups related playback attempts by user and client. | | `hls_access_lease_id` | Per playback URL | Random server-side lease ID. | Lets exactly one playback path use the shared session after Tuliprox validates the user and lease. | | HLS cache user session token | Per playback admission | Generated for the Shared HLS entry request. | Connects the access lease to Tuliprox user session accounting and connection handling. | -The key design point is that `proxy_session_id` is shared, but `hls_access_lease_id` is not. +The key design point is that `proxy_session_id` is shared, but `hls_access_lease_id` is not. The `stream_ref` field used +inside `HlsSessionKey` is exactly the original `input_stream_id`: an Xtream provider/origin stream ID as a decimal string, +or the parser-resolved M3U ID, including alphanumeric IDs and stable URL-derived hashes. `target_id` and `virtual_id` stay +in the routing and access context and do not affect this shared identity. ## End-to-end request flow @@ -50,8 +53,9 @@ sequenceDiagram ## Step 1: Entry request -The player first opens a generated Tuliprox live HLS URL. Tuliprox resolves the target, input, virtual ID, stream URL, and -user connection permission. +The player first opens a generated Tuliprox live HLS URL. Tuliprox resolves the target, input, virtual ID, original input +stream ID, stream URL, and user connection permission. The original input stream ID was captured before target mapping +and is not replaced by the target's virtual ID. If Shared HLS is enabled for the target, Tuliprox creates a fresh `Pending` `HlsAccessLease` for this playback and returns an HTTP `307 Temporary Redirect` to the canonical shared manifest path: @@ -213,3 +217,8 @@ Typical outcomes are: - `503 Service Unavailable` with `Retry-After` when origin/cache work is temporarily not ready. Players should reopen the original generated playlist entry URL when they recover from a long pause or stale HLS path. + +An upgrade from `virtual_id`-based session identity also makes old canonical URLs stale when the virtual and input stream +IDs differ. New entry requests receive the new `proxy_session_id`; old cache namespaces are left for normal garbage +collection. Persisted M3U-to-Xtream data that lost a nonnumeric input ID as `provider_id = 0` must be refreshed before the +entry can safely join a shared session. diff --git a/docs/src/configuration/shared-hls-sessions.md b/docs/src/configuration/shared-hls-sessions.md index 48a001f9f..79d5ee39d 100644 --- a/docs/src/configuration/shared-hls-sessions.md +++ b/docs/src/configuration/shared-hls-sessions.md @@ -43,7 +43,7 @@ separate `share_live_streams.mpeg_ts` option. | Shared between viewers | Still separate per viewer | | :--- | :--- | -| The stable `HlsSession` for the same input and live stream reference. | The `hls_access_lease_id` in the user-facing URL. | +| The stable `HlsSession` for the same input and original input stream ID. | The `hls_access_lease_id` in the user-facing URL. | | Origin manifest refreshes and origin segment fetches when the cache can reuse them. | Tuliprox username, client fingerprint, and access checks. | | Cached HLS segments and HLS MAP objects. | User connection admission and stream reservation lifecycle. | | Transient HLS resource handling when the manifest requires it. | User-visible error handling and custom video redirects. | @@ -51,6 +51,26 @@ separate `share_live_streams.mpeg_ts` option. This means the feature is not a bypass for user limits. It is a way to avoid duplicated upstream HLS work after Tuliprox has accepted each viewer. +## How is a shared session identified? + +Tuliprox builds the shared content identity as: + +```text +input:|hls| +``` + +`input_id` identifies the configured Tuliprox input. `input_stream_id` is the original stream ID captured from the input +playlist item before target mapping. For Xtream inputs, it is the provider/origin stream ID rendered as a decimal string. +For M3U inputs, it is the exact ID resolved by the parser, which can also be alphanumeric or a stable URL-derived hash. + +The target ID and `virtual_id` are not part of this key. Consequently, two targets with different virtual IDs share the +same content session when they refer to the same `input_id` and `input_stream_id`. Their access leases, routing context, +and user admission remain separate. Origin credentials, direct origin URLs, and the currently selected provider mirror +also do not change the shared session identity. + +The pair `(input_id, input_stream_id)` must uniquely identify one stream within the processed inputs. Target mapping must +not change the captured input stream ID. + ## Requirements Shared HLS starts only when both switches are enabled: @@ -98,7 +118,8 @@ targets: ## Request flow in plain language 1. A player opens the normal generated Tuliprox HLS live URL. -2. Tuliprox authenticates the user and checks whether this target allows Shared HLS. +2. Tuliprox authenticates the user, resolves the unchanged input stream ID, and checks whether this target allows Shared + HLS. 3. Tuliprox creates a new access lease for that playback. 4. Tuliprox returns an HTTP `307 Temporary Redirect` to the canonical shared HLS manifest URL. 5. The player follows the redirect and requests `/hls/shared/live/.../manifest.m3u8`. diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index b66f375c4..aa8a7d426 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -278,6 +278,8 @@ pub trait FieldSetAccessor { pub trait PlaylistEntry: Send + Sync { fn get_virtual_id(&self) -> VirtualId; + /// Returns the immutable stream identifier captured from the input playlist before target mappings. + fn get_input_stream_id(&self) -> Option>; fn get_provider_id(&self) -> Option; fn get_category_id(&self) -> Option; fn get_provider_url(&self) -> Arc; @@ -335,6 +337,9 @@ pub struct PlaylistItemHeader { pub xtream_cluster: XtreamCluster, #[serde(default)] pub item_type: PlaylistItemType, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, } impl Default for PlaylistItemHeader { @@ -361,11 +366,30 @@ impl Default for PlaylistItemHeader { category_id: 0, input_name: "".intern(), source_ordinal: 0, + input_stream_id: "".intern(), } } } impl PlaylistItemHeader { + /// Captures the input playlist item ID at the input-processing boundary. + /// + /// This must run before target transformations and must not be used to recover an identity + /// from a target-mutated `id`. + pub fn freeze_input_stream_id(&mut self) { + if self.input_stream_id.is_empty() && !self.id.is_empty() { + self.input_stream_id = Arc::clone(&self.id); + } + } + + /// Returns the input stream ID captured at the input-processing boundary. + /// + /// Legacy fallback must be resolved before a target transformation starts. Once an item is + /// represented by this header, the mutable target-facing `id` is never a valid fallback. + pub fn get_input_stream_id(&self) -> Option> { + (!self.input_stream_id.is_empty()).then(|| Arc::clone(&self.input_stream_id)) + } + #[inline] pub fn gen_uuid(&mut self) { self.uuid = generate_runtime_playlist_uuid(&self.input_name, &self.id, self.item_type, &self.url); @@ -544,6 +568,9 @@ pub struct M3uPlaylistItem { pub source_ordinal: u32, #[serde(default)] pub additional_properties: Option, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, } fn write_m3u_attr(line: &mut String, name: &str, value: &str) { let _ = write!(line, " {name}=\"{value}\""); } @@ -660,6 +687,11 @@ impl PlaylistEntry for M3uPlaylistItem { #[inline] fn get_virtual_id(&self) -> VirtualId { self.virtual_id } + fn get_input_stream_id(&self) -> Option> { + let input_stream_id = if self.input_stream_id.is_empty() { &self.provider_id } else { &self.input_stream_id }; + (!input_stream_id.is_empty()).then(|| Arc::clone(input_stream_id)) + } + fn get_provider_id(&self) -> Option { get_provider_id(&self.provider_id, &self.url) } #[inline] fn get_category_id(&self) -> Option { None } @@ -884,6 +916,9 @@ pub struct XtreamPlaylistItem { pub channel_no: u32, #[serde(default)] pub source_ordinal: u32, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, } impl XtreamPlaylistItem { @@ -940,6 +975,13 @@ impl XtreamPlaylistItem { impl PlaylistEntry for XtreamPlaylistItem { #[inline] fn get_virtual_id(&self) -> VirtualId { self.virtual_id } + fn get_input_stream_id(&self) -> Option> { + if self.input_stream_id.is_empty() { + (self.provider_id > 0).then(|| self.provider_id.to_string().intern()) + } else { + Some(Arc::clone(&self.input_stream_id)) + } + } #[inline] fn get_provider_id(&self) -> Option { Some(self.provider_id) } #[inline] @@ -1166,7 +1208,11 @@ impl PlaylistItem { impl From<&PlaylistItem> for XtreamPlaylistItem { fn from(item: &PlaylistItem) -> Self { let header = &item.header; - let provider_id = get_provider_id(&header.id, &header.url).unwrap_or_default(); + let input_stream_id = Arc::clone(&header.input_stream_id); + let missing_live_input_identity = + input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); + let provider_id = + if missing_live_input_identity { 0 } else { get_provider_id(&header.id, &header.url).unwrap_or_default() }; let additional_properties = PlaylistItem::get_additional_properties(header); XtreamPlaylistItem { @@ -1192,6 +1238,7 @@ impl From<&PlaylistItem> for XtreamPlaylistItem { input_name: Arc::clone(&header.input_name), channel_no: header.chno, source_ordinal: header.source_ordinal, + input_stream_id, } } } @@ -1199,9 +1246,12 @@ impl From<&PlaylistItem> for XtreamPlaylistItem { impl From<&PlaylistItem> for M3uPlaylistItem { fn from(item: &PlaylistItem) -> Self { let header = &item.header; + let input_stream_id = Arc::clone(&header.input_stream_id); + let missing_live_input_identity = + input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); M3uPlaylistItem { virtual_id: header.virtual_id, - provider_id: Arc::clone(&header.id), + provider_id: if missing_live_input_identity { "".intern() } else { Arc::clone(&header.id) }, name: if header.item_type == PlaylistItemType::Series { Arc::clone(&header.title) } else { @@ -1226,6 +1276,7 @@ impl From<&PlaylistItem> for M3uPlaylistItem { t_catchup_mode: None, source_ordinal: header.source_ordinal, additional_properties: header.additional_properties.clone(), + input_stream_id, } } } @@ -1266,10 +1317,15 @@ impl From<&PlaylistItem> for CommonPlaylistItem { impl From<&XtreamPlaylistItem> for PlaylistItem { fn from(item: &XtreamPlaylistItem) -> Self { + let input_stream_id = item.get_input_stream_id(); let header = PlaylistItemHeader { uuid: item.get_uuid(), virtual_id: item.virtual_id, - id: item.provider_id.to_string().intern(), + id: if item.provider_id == 0 && input_stream_id.is_none() { + "".intern() + } else { + item.provider_id.to_string().intern() + }, name: item.name.clone(), title: item.title.clone(), logo: item.logo.clone(), @@ -1288,6 +1344,7 @@ impl From<&XtreamPlaylistItem> for PlaylistItem { time_shift: "".intern(), additional_properties: item.additional_properties.clone(), source_ordinal: item.source_ordinal, + input_stream_id: input_stream_id.unwrap_or_else(|| "".intern()), }; PlaylistItem { header } @@ -1318,6 +1375,7 @@ impl From<&M3uPlaylistItem> for PlaylistItem { time_shift: item.time_shift.clone(), additional_properties: item.additional_properties.clone(), source_ordinal: item.source_ordinal, + input_stream_id: item.get_input_stream_id().unwrap_or_else(|| "".intern()), }; PlaylistItem { header } @@ -1328,6 +1386,9 @@ impl PlaylistEntry for PlaylistItem { #[inline] fn get_virtual_id(&self) -> VirtualId { self.header.virtual_id } + #[inline] + fn get_input_stream_id(&self) -> Option> { self.header.get_input_stream_id() } + fn get_provider_id(&self) -> Option { let header = &self.header; get_provider_id(&header.id, &header.url) @@ -1389,6 +1450,7 @@ impl PlaylistGroup { #[inline] pub fn on_load(&mut self) { for pl in &mut self.channels { + pl.header.freeze_input_stream_id(); pl.header.gen_uuid(); pl.header.category_id = self.id; } @@ -1554,7 +1616,7 @@ mod tests { #[test] fn xtream_playlist_item_conversion_falls_back_to_numeric_id_in_url() { - let item = PlaylistItem { + let mut item = PlaylistItem { header: PlaylistItemHeader { id: "channel-alpha".intern(), url: "http://provider.example/live/user/pass/12345.ts".intern(), @@ -1564,16 +1626,142 @@ mod tests { ..PlaylistItemHeader::default() }, }; + item.header.freeze_input_stream_id(); let xtream_item = XtreamPlaylistItem::from(&item); assert_eq!(xtream_item.provider_id, 12345); } + #[test] + fn xtream_playlist_item_preserves_numeric_input_stream_id_as_string() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + virtual_id: 1001, + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + item.header.freeze_input_stream_id(); + + let xtream_item = XtreamPlaylistItem::from(&item); + + assert_eq!(xtream_item.provider_id, 80510); + assert_eq!(xtream_item.input_stream_id.as_ref(), "80510"); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); + } + + #[test] + fn alphanumeric_m3u_input_stream_id_survives_xtream_materialization() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "channel-alpha".intern(), + url: "http://provider.example/live/user/pass/12345.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + + let m3u_item = M3uPlaylistItem::from(&source); + let common_item = PlaylistItem::from(&m3u_item); + let xtream_item = XtreamPlaylistItem::from(&common_item); + + assert_eq!(m3u_item.input_stream_id.as_ref(), "channel-alpha"); + assert_eq!(xtream_item.provider_id, 12345); + assert_eq!(xtream_item.input_stream_id.as_ref(), "channel-alpha"); + } + + #[test] + fn url_hash_input_stream_id_is_preserved_verbatim() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "d34db33f".intern(), + url: "http://provider.example/live/channel.m3u8".intern(), + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + + let m3u_item = M3uPlaylistItem::from(&source); + + assert_eq!(m3u_item.input_stream_id.as_ref(), "d34db33f"); + } + + #[test] + fn target_field_mapping_cannot_change_frozen_input_stream_id() { + let mut header = PlaylistItemHeader { id: "origin-alpha".intern(), ..PlaylistItemHeader::default() }; + header.freeze_input_stream_id(); + + assert!(header.set_field("id", "target-id")); + + assert_eq!(header.id.as_ref(), "target-id"); + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); + assert!(!header.set_field("input_stream_id", "unexpected")); + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); + } + + #[test] + fn legacy_playlist_items_fall_back_to_provider_id_without_virtual_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "legacy-alpha".intern(), + virtual_id: 7001, + url: "http://provider.example/live/user/pass/80510.ts".intern(), + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + let mut m3u_item = M3uPlaylistItem::from(&source); + m3u_item.input_stream_id = "".intern(); + let mut xtream_item = XtreamPlaylistItem::from(&source); + xtream_item.input_stream_id = "".intern(); + + assert_eq!(m3u_item.get_input_stream_id().as_deref(), Some("legacy-alpha")); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); + xtream_item.provider_id = 0; + xtream_item.url = "http://provider.example/live/channel.ts".intern(); + assert_eq!(xtream_item.get_input_stream_id(), None); + + let mut legacy_common_item = PlaylistItem::from(&xtream_item); + assert!(legacy_common_item.header.id.is_empty()); + assert_eq!(legacy_common_item.get_input_stream_id(), None); + legacy_common_item.header.id = "target-mapped-id".intern(); + let rematerialized_m3u_item = M3uPlaylistItem::from(&legacy_common_item); + let rematerialized_xtream_item = XtreamPlaylistItem::from(&legacy_common_item); + assert_eq!(legacy_common_item.header.id.as_ref(), "target-mapped-id"); + assert_eq!(legacy_common_item.get_input_stream_id(), None); + assert!(rematerialized_m3u_item.provider_id.is_empty()); + assert_eq!(rematerialized_m3u_item.get_input_stream_id(), None); + assert_eq!(rematerialized_xtream_item.provider_id, 0); + assert_eq!(rematerialized_xtream_item.get_input_stream_id(), None); + + xtream_item.input_stream_id = "explicit-origin-alpha".intern(); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + let identified_common_item = PlaylistItem::from(&xtream_item); + assert_eq!(identified_common_item.header.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_common_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + + let identified_m3u_item = M3uPlaylistItem::from(&identified_common_item); + let identified_xtream_item = XtreamPlaylistItem::from(&identified_common_item); + assert_eq!(identified_m3u_item.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_m3u_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + assert_eq!(identified_xtream_item.provider_id, 0); + assert_eq!(identified_xtream_item.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + } + #[test] fn m3u_to_m3u_emits_tvg_id_from_epg_channel_id() { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1607,6 +1795,7 @@ mod tests { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1638,6 +1827,7 @@ mod tests { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 42, logo: "".intern(), @@ -1669,6 +1859,7 @@ mod tests { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1700,6 +1891,7 @@ mod tests { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1747,6 +1939,7 @@ mod tests { let item = M3uPlaylistItem { virtual_id: 0, provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), name: "Test Channel".intern(), chno: 0, logo: "".intern(), diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index b542bfc73..49c512137 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -383,6 +383,7 @@ mod tests { input_name: "demo".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "1".intern(), }; let stream_channel = create_stream_channel_with_type(1, &playlist_item, PlaylistItemType::Video); @@ -412,6 +413,7 @@ mod tests { input_name: "provider-input".intern(), channel_no: 0, source_ordinal: 0, + input_stream_id: "1".intern(), }; let stream_channel = playlist_item.to_stream_channel(1); @@ -444,6 +446,7 @@ mod tests { t_catchup_mode: None, source_ordinal: 0, additional_properties: None, + input_stream_id: "42".intern(), }; let channel = pli.to_stream_channel(7).with_epg_reference_ts(Some(1_700_000_000)); @@ -479,6 +482,7 @@ mod tests { t_catchup_mode: None, source_ordinal: 0, additional_properties: None, + input_stream_id: "42".intern(), }; // to_stream_channel() defaults to None, and with_epg_reference_ts(None) diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index 6e040d0c4..9f5730a19 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -1051,6 +1051,7 @@ mod tests { input_name: "input".into(), channel_no: 0, source_ordinal: 0, + input_stream_id: "1001".into(), } } diff --git a/shared/src/utils/constants.rs b/shared/src/utils/constants.rs index 86e0c9ee5..a957a8dbf 100644 --- a/shared/src/utils/constants.rs +++ b/shared/src/utils/constants.rs @@ -15,9 +15,6 @@ pub const DEFAULT_HOME_ENV_VAR: &str = "TULIPROX_HOME"; pub const DEFAULT_WEB_ROOT_ENV_VAR: &str = "TULIPROX_WEB_ROOT"; pub const DNS_RESOLVED_FILE: &str = "provider_dns_resolved.json"; -pub const ENCODING_GZIP: &str = "gzip"; -pub const ENCODING_DEFLATE: &str = "deflate"; - pub const FILENAME_TRIM_PATTERNS: &[char] = &['.', '-', '_']; const SUPPORTED_RESPONSE_HEADERS: &[&str] = &[ @@ -27,7 +24,6 @@ const SUPPORTED_RESPONSE_HEADERS: &[&str] = &[ "content-length", "content-range", "vary", - "transfer-encoding", //"connection", "access-control-allow-origin", "access-control-allow-credentials",