diff --git a/.gitignore b/.gitignore index b6480f0b2..df0e3dfd5 100644 --- a/.gitignore +++ b/.gitignore @@ -56,8 +56,4 @@ backend/Cargo.lock !CONTRIBUTING.md # Ignore ai documentation files -.c[lor][adu]* -/.dual-graph* -/graphify* -.dual-graph/ -.dual-graph-context/ +# no ai files diff --git a/CHANGELOG.md b/CHANGELOG.md index 3452ba438..2e82e831c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -326,6 +326,11 @@ - **Stream View**: - Displays the user comment in the stream view. - Displays EPG information in the stream view. +- **Shared HLS Streams** + - Added advanced video routing with HLS and TS support, including HLS provisioning polling and correct byte-range + handling for TS output. + - Introduced live HLS reverse-proxy caching with lifecycle scheduling, garbage collection, and improved demand/prefetch backpressure. + - Added smarter HLS playback access/admission handling for consistent manifest responses. ## 🐛 Fixes diff --git a/Cargo.lock b/Cargo.lock index 453f4383f..1eaf18912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4663,6 +4663,7 @@ dependencies = [ "serde-saphyr", "serde_html_form", "serde_json", + "sha2", "shared", "smallvec", "socket2", diff --git a/Cargo.toml b/Cargo.toml index bcf29f463..4686ca82d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] version = "3.3.56" edition = "2021" -rust-version = "1.89.0" +rust-version = "1.95.0" [workspace.dependencies] serde = { version = "1.0.228", features = ["derive", "rc"] } @@ -32,6 +32,7 @@ url = "2.5.8" cron = "0.16.0" futures = "0.3.32" serde-saphyr = "0.0.26" +sha2 = "0.10.9" thiserror = "2.0.18" tokio = { version = "1.52.3" } strum = "0.28.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 1a6307ad9..9369c085e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -58,6 +58,7 @@ strsim = "0.11.1" rphonetic = "3.0.6" rayon = "1.12.0" base64.workspace = true +sha2.workspace = true paste.workspace = true indexmap.workspace = true parking_lot = "0.12.5" diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index e2fdbf783..21ec256a7 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -7,8 +7,8 @@ use crate::{ get_custom_stream_response_error_status, get_stream_response_with_headers, is_custom_video_stream_enabled, tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamFactoryOptions, ProviderStreamInfo, - ProviderStreamState, SharedStreamManager, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, - UserApiRequest, UserSession, + ProviderStreamCustomReason, ProviderStreamState, SharedStreamManager, StreamDetails, StreamError, + StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, }, }, auth::Fingerprint, @@ -45,13 +45,14 @@ use shared::{ concat_string, model::{ Claims, InputFetchMethod, InputType, PlaylistEntry, PlaylistItemType, ProxyType, StreamChannel, StreamInfo, TargetType, - UserConnectionPermission, VirtualId, XtreamCluster, + UserConnectionPermission, VirtualId, XtreamCluster, ConfigTargetOptions, }, utils::{ bin_serialize, current_time_secs, extract_extension_from_url, get_credentials_from_url, human_readable_kbps, is_sanitize_sensitive_info_enabled, replace_url_extension, sanitize_sensitive_info, trim_slash, Internable, - CONTENT_TYPE_CBOR, CONTENT_TYPE_JSON, DASH_EXT, HLS_EXT, + CONTENT_TYPE_CBOR, CONTENT_TYPE_JSON, }, + defaults::{DASH_EXT, HLS_EXT,}, }; use smallvec::SmallVec; use std::{ @@ -479,7 +480,7 @@ use crate::utils::LRUResourceCache; pub use internal_server_error; use shared::error::TuliproxError; use shared::model::{AdmissionStrategy, ConnectFailureReason, FailureStage, GeoIpUnavailablePolicy}; -use shared::utils::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}; +use shared::defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}; pub use try_option_bad_request; pub use try_option_forbidden; pub use try_result_bad_request; @@ -1638,11 +1639,10 @@ fn select_provider_stream_url( } fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamState { - ProviderStreamState::Custom(create_channel_unavailable_stream( - app_config, - &[], - StatusCode::SERVICE_UNAVAILABLE, - )) + ProviderStreamState::Custom { + response: create_channel_unavailable_stream(app_config, &[], StatusCode::OK), + reason: ProviderStreamCustomReason::UnmappedProviderUrl, + } } async fn acquire_stream_provider_handle( @@ -1776,7 +1776,10 @@ async fn resolve_streaming_strategy( ProviderAllocation::Exhausted => { debug!("Provider {} is exhausted. No connections allowed.", input.name); let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom(stream) + ProviderStreamState::Custom { + response: stream, + reason: ProviderStreamCustomReason::ProviderExhausted, + } } ProviderAllocation::Available(ref provider_cfg) | ProviderAllocation::GracePeriod(ref provider_cfg) => { // Keep the URL only when it already targets the selected provider account. Hot reload can leave old @@ -1819,7 +1822,10 @@ async fn resolve_streaming_strategy( } else { debug!("Provider {} is exhausted. No connections allowed.", input.name); let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom(stream) + ProviderStreamState::Custom { + response: stream, + reason: ProviderStreamCustomReason::ProviderExhausted, + } }; if release_failed_mapping { @@ -1931,7 +1937,13 @@ async fn create_stream_response_details( let guard_provider_name = streaming_strategy.provider_handle.as_ref().and_then(|guard| guard.allocation.get_provider_name()); - if matches!(streaming_strategy.provider_stream_state, ProviderStreamState::Custom(_)) + if matches!( + streaming_strategy.provider_stream_state, + ProviderStreamState::Custom { + reason: ProviderStreamCustomReason::ProviderExhausted, + .. + } + ) && can_provision_on_exhausted(app_state, input) { if let Some(handle) = streaming_strategy.provider_handle.take() { @@ -1953,7 +1965,10 @@ async fn create_stream_response_details( match streaming_strategy.provider_stream_state { // custom stream means we display our own stream like connection exhausted, channel-unavailable... - ProviderStreamState::Custom(provider_stream) => { + ProviderStreamState::Custom { + response: provider_stream, + .. + } => { let (stream, stream_info) = provider_stream; // When allocation is exhausted or no connection was acquired, guard_provider_name is None. // Use input.name as fallback so the provider field is never empty. @@ -2317,11 +2332,19 @@ fn is_hop_by_hop_response_header(name: &HeaderName) -> bool { } fn no_custom_video_fallback_status(app_config: &AppConfig) -> StatusCode { + // Two reasons we have no custom-video response: + // 1. Operator disabled `custom_stream_response_enabled` → return the + // configured fallback status (e.g. 502) so reverse proxies handle the + // socket consistently. + // 2. Operator enabled custom-video but the concrete resource is missing + // → return `400` so downstream `proxy_intercept_errors on;` (Nginx) + // can sever the socket instead of looping on `200 OK`. + // Collapsing both into the configured status code broke the Nginx-intercept + // contract that the operator relied on by enabling custom-video in the + // first place. if is_custom_video_stream_enabled(app_config) { - // No custom video response available but custom video response enabled StatusCode::BAD_REQUEST } else { - // reverse proxy with `proxy_intercept_errors on;` can sever the socket. get_custom_stream_response_error_status(app_config) } } @@ -3344,6 +3367,7 @@ pub(crate) async fn local_stream_response( }) .await; } + let metering = prepare_stream_metering(app_state, &pli.url, false, true, false).await; let stream = create_active_client_stream(crate::api::model::ActiveClientStreamParams { stream_details: StreamDetails::from_stream(stream, grace_period_options), app_state, @@ -3355,8 +3379,8 @@ pub(crate) async fn local_stream_response( socket_bound, session_token: playback_session_token, req_headers, - meter_uid: 0, - meter_stream: false, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, }) .await; @@ -3397,7 +3421,11 @@ fn is_path_within_allowed_directories(sub_path: &Path, root_paths: &[String]) -> pub fn is_stream_share_enabled(item_type: PlaylistItemType, target: &ConfigTarget) -> bool { (item_type == PlaylistItemType::Live/* || item_type == PlaylistItemType::LiveHls */) - && target.options.as_ref().is_some_and(|opt| opt.share_live_streams) + && target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_mpeg_ts_enabled) +} + +pub fn is_hls_stream_share_enabled(target: &ConfigTarget) -> bool { + target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_hls_enabled) } pub type HeaderFilter = Option bool + Send>>; @@ -4071,7 +4099,8 @@ mod tests { ClusterFlags, ConfigPaths, ConfigProviderDto, ConfigTargetOptions, InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder, ProviderUrlSelectionPolicy, ProxyType, StreamChannel, XtreamCluster, }, - utils::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs, Internable}, + utils::{Internable}, + defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, }; use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; use tokio::sync::{mpsc, RwLock}; @@ -4727,7 +4756,13 @@ mod tests { .await; assert!(strict.provider_handle.is_none(), "strict provider affinity should not allocate a different provider"); assert!( - matches!(strict.provider_stream_state, ProviderStreamState::Custom(_)), + matches!( + strict.provider_stream_state, + ProviderStreamState::Custom { + reason: ProviderStreamCustomReason::ProviderExhausted, + .. + } + ), "strict provider affinity should fail closed when the pinned provider is unavailable" ); @@ -4829,7 +4864,13 @@ mod tests { .await; assert!(strategy.provider_handle.is_none()); - assert!(matches!(strategy.provider_stream_state, ProviderStreamState::Custom(_))); + assert!(matches!( + strategy.provider_stream_state, + ProviderStreamState::Custom { + reason: ProviderStreamCustomReason::UnmappedProviderUrl, + .. + } + )); app_state.active_provider.release_connection(&addr).await; } @@ -5092,6 +5133,8 @@ mod tests { downloads: Arc::new(crate::api::model::DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(crate::api::model::HlsProxyManager::new()), + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), active_users, active_provider, connection_manager, @@ -5126,6 +5169,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }), user_access_control: true, ..Config::default() @@ -5882,6 +5926,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }), ..Config::default() })); @@ -5985,6 +6030,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }), ..Config::default() })); @@ -6040,7 +6086,13 @@ mod tests { id: 1, enabled: true, name: "shared".to_string(), - options: Some(ConfigTargetOptions { share_live_streams: true, ..ConfigTargetOptions::default() }), + options: Some(ConfigTargetOptions { + share_live_streams: shared::model::ConfigTargetShareLiveStreams { + mpeg_ts: true, + ..Default::default() + }, + ..ConfigTargetOptions::default() + }), sort: None, filter: Filter::default(), output: Vec::new(), @@ -7714,6 +7766,10 @@ mod tests { .await .into_response(); + // Custom-video stream is enabled (`custom_stream_response_enabled: true` + // in this fixture), so a missing resource must return 400 — the + // Nginx `proxy_intercept_errors on;` contract requires 4xx so the + // socket is severed instead of looping on a 200 OK fallback body. assert_eq!(response.status(), StatusCode::BAD_REQUEST); assert_eq!( app_state.active_users.user_connections(&user.username).await, @@ -8160,7 +8216,7 @@ mod tests { } #[tokio::test] - async fn socket_bound_playback_tokens_enforce_hard_limits_per_socket() { + async fn socket_bound_playback_sessions_enforce_hard_limits_per_socket() { let app_state = create_test_app_state(); let mut user = ProxyUserCredentials::default(); user.username = "user1".to_string(); @@ -8336,7 +8392,7 @@ mod tests { } #[tokio::test] - async fn socket_bound_playback_tokens_still_allow_soft_slots() { + async fn socket_bound_playback_sessions_still_allow_soft_slots() { let app_state = create_test_app_state(); let mut user = ProxyUserCredentials::default(); user.username = "soft-user".to_string(); @@ -8535,6 +8591,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }), ..Config::default() }; @@ -8815,14 +8872,14 @@ mod tests { source_ordinal: 0, }; - let hls_ext = shared::utils::HLS_EXT.to_string(); + let hls_ext = shared::defaults::HLS_EXT.to_string(); let (query_path, extension) = crate::api::endpoints::xtream_api::get_query_path("", Some(&hls_ext), &pli, &app_state); assert_eq!(extension, ""); assert_eq!(query_path, "1"); - let dash_ext = shared::utils::DASH_EXT.to_string(); + let dash_ext = shared::defaults::DASH_EXT.to_string(); let (query_path, extension) = crate::api::endpoints::xtream_api::get_query_path("", Some(&dash_ext), &pli, &app_state); diff --git a/backend/src/api/endpoints/custom_video_stream_api.rs b/backend/src/api/endpoints/custom_video_stream_api.rs index be0bc4eb7..e8d57fb76 100644 --- a/backend/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/src/api/endpoints/custom_video_stream_api.rs @@ -1,52 +1,552 @@ use crate::{ - api::{api_utils::create_api_proxy_user, model::{create_custom_video_stream_response, AppState, CustomVideoStreamType}}, - auth::{verify_access_token, Fingerprint}, + api::{ + api_utils::{create_api_proxy_user, mark_response_as_uncompressed}, + endpoints::hls_api::{ + build_virtual_hls_entry_path, hls_panel_provisioning_poll_manifest_response, + resolve_hls_virtual_input_for_target, + }, + model::{ + create_custom_video_stream_response, hls_custom_video_manifest_response_with_virtual_id, + parse_hls_panel_provisioning_segment_route_name, AppState, CustomVideoStreamType, + TransportStreamBuffer, + }, + }, + auth::{check_network_access_only, resolve_api_user_context, verify_access_token, Fingerprint}, + model::{ConfigTarget, ProxyUserCredentials}, }; -use axum::response::IntoResponse; +use axum::{ + body::Body, + http::{ + header::{self, HeaderName}, + HeaderMap, HeaderValue, StatusCode, + }, + response::{IntoResponse, Response}, +}; +use serde::Deserialize; use std::{str::FromStr, sync::Arc}; -use crate::auth::resolve_api_user_context; use url::form_urlencoded; +const HLS_CVS_CONTENT_TYPE: &str = "video/mp2t"; +const HLS_CVS_MEDIA_EXTENSIONS: &[&str] = &["ts", "mp4", "m4s", "m4v"]; +const ACCEPT_RANGES_VALUE: &str = "bytes"; +const HLS_CVS_CACHE_CONTROL: &str = "no-store"; + +#[derive(Debug, Deserialize)] +struct ProvisioningManifestQuery { + id: u32, +} + +type CvsUserContext = (Arc, Arc); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CvsRouteKind { + Hls, + Ts, +} + +#[derive(Clone, Copy)] +struct CvsApiResponseContext<'a> { + fingerprint: &'a Fingerprint, + username: &'a str, + password: &'a str, + stream_type: &'a str, + route_kind: CvsRouteKind, + request_headers: &'a HeaderMap, + raw_query: Option<&'a str>, + app_state: &'a Arc, +} + +fn resolve_cvs_user_context( + app_state: &Arc, + fingerprint: &Fingerprint, + username: &str, + password: &str, +) -> Result> { + let Some((user, target)) = app_state.app_config.get_target_for_user(username, password) else { + return Err(Box::new(app_state.app_config.get_auth_error_status().into_response())); + }; + + if let Err(e) = resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), app_state) { + return Err(Box::new(e.into_player_response(app_state.app_config.get_auth_error_status()))); + } + + Ok((user, target)) +} + +fn resolve_hls_cvs_user_context( + app_state: &Arc, + fingerprint: &Fingerprint, + username: &str, + password: &str, +) -> Result> { + let Some((user, target)) = app_state.app_config.get_target_for_user(username, password) else { + return Err(Box::new(app_state.app_config.get_auth_error_status().into_response())); + }; + + if let Err(e) = check_network_access_only(&user, fingerprint, app_state) { + return Err(Box::new(e.into_player_response(app_state.app_config.get_auth_error_status()))); + } + + Ok((user, target)) +} + +async fn cvs_typed_api( + fingerprint: Fingerprint, + axum::extract::Path((route_kind, username, password, stream_type)): axum::extract::Path<( + String, + String, + String, + String, + )>, + axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + headers: HeaderMap, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let route_kind = match route_kind.as_str() { + "hls" => CvsRouteKind::Hls, + "ts" => CvsRouteKind::Ts, + _ => return axum::http::StatusCode::NOT_FOUND.into_response(), + }; + cvs_api_response(CvsApiResponseContext { + fingerprint: &fingerprint, + username: &username, + password: &password, + stream_type: &stream_type, + route_kind, + request_headers: &headers, + raw_query: raw_query.as_deref(), + app_state: &app_state, + }) +} + async fn cvs_api( fingerprint: Fingerprint, axum::extract::Path((username, password, stream_type)): axum::extract::Path<(String, String, String)>, axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + headers: HeaderMap, axum::extract::State(app_state): axum::extract::State>, ) -> impl IntoResponse + Send { - let cvs_type = stream_type.strip_suffix(".ts").unwrap_or(&stream_type); + cvs_api_response(CvsApiResponseContext { + fingerprint: &fingerprint, + username: &username, + password: &password, + stream_type: &stream_type, + route_kind: CvsRouteKind::Ts, + request_headers: &headers, + raw_query: raw_query.as_deref(), + app_state: &app_state, + }) +} + +fn cvs_api_response(context: CvsApiResponseContext<'_>) -> Response { + let CvsApiResponseContext { + fingerprint, + username, + password, + stream_type, + route_kind, + request_headers, + raw_query, + app_state, + } = context; + + if route_kind == CvsRouteKind::Hls { + if let Some(index) = parse_hls_panel_provisioning_segment_route_name(stream_type) { + if let Err(response) = resolve_cvs_user_context(app_state, fingerprint, username, password) { + return *response; + } + return create_hls_provisioning_segment_response(app_state, request_headers, index); + } + } + + if route_kind == CvsRouteKind::Hls { + if let Some(cvs_type) = stream_type.strip_suffix(".m3u8") { + let Ok(custom_video_type) = CustomVideoStreamType::from_str(cvs_type) else { + return axum::http::StatusCode::NOT_FOUND.into_response(); + }; + let (user, _) = match resolve_hls_cvs_user_context(app_state, fingerprint, username, password) { + Ok(context) => context, + Err(response) => return *response, + }; + return hls_custom_video_manifest_response_with_virtual_id( + app_state, + &user, + custom_video_type, + StatusCode::NOT_FOUND, + None, + ); + } + } + + let cvs_type = strip_hls_custom_video_media_extension(stream_type); let Ok(custom_video_type) = CustomVideoStreamType::from_str(cvs_type) else { return axum::http::StatusCode::NOT_FOUND.into_response(); }; - let api_proxy_user = create_api_proxy_user(&app_state); - if username == api_proxy_user.username && password == api_proxy_user.password { - let token = raw_query.as_deref().and_then(|query| { - form_urlencoded::parse(query.as_bytes()) - .find_map(|(key, value)| (key == "token").then(|| value.into_owned())) - }); - let Some(token) = token.as_deref() else { - return app_state.app_config.get_auth_error_status().into_response(); - }; - if !verify_access_token(token, &app_state.app_config.access_token_secret) { - return app_state.app_config.get_auth_error_status().into_response(); + if route_kind == CvsRouteKind::Ts { + let api_proxy_user = create_api_proxy_user(app_state); + if username == api_proxy_user.username && password == api_proxy_user.password { + let token = raw_query.and_then(|query| { + form_urlencoded::parse(query.as_bytes()) + .find_map(|(key, value)| (key == "token").then(|| value.into_owned())) + }); + let Some(token) = token.as_deref() else { + return app_state.app_config.get_auth_error_status().into_response(); + }; + if !verify_access_token(token, &app_state.app_config.access_token_secret) { + return app_state.app_config.get_auth_error_status().into_response(); + } + return create_custom_video_stream_response(app_state, &fingerprint.addr, custom_video_type).into_response(); } - return create_custom_video_stream_response(&app_state, &fingerprint.addr, custom_video_type).into_response(); } - let Some((user, target)) = app_state.app_config.get_target_for_user(&username, &password) else { - return app_state.app_config.get_auth_error_status().into_response(); + let auth_result = match route_kind { + CvsRouteKind::Hls => resolve_hls_cvs_user_context(app_state, fingerprint, username, password), + CvsRouteKind::Ts => resolve_cvs_user_context(app_state, fingerprint, username, password), + }; + if let Err(response) = auth_result { + return *response; + } + + match route_kind { + CvsRouteKind::Hls => create_hls_custom_video_segment_response(app_state, request_headers, custom_video_type), + CvsRouteKind::Ts => create_custom_video_stream_response(app_state, &fingerprint.addr, custom_video_type) + .into_response(), + } +} + +fn strip_hls_custom_video_media_extension(stream_type: &str) -> &str { + stream_type + .rsplit_once('.') + .filter(|(_, extension)| HLS_CVS_MEDIA_EXTENSIONS.contains(extension)) + .map_or(stream_type, |(raw, _)| raw) +} + +fn hls_provisioning_segment_buffer(app_state: &Arc, index: usize) -> Option { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + custom_stream_response + .as_ref() + .and_then(|response| response.panel_api_provisioning_hls_segments.get(index).cloned()) +} + +fn create_hls_provisioning_segment_response( + app_state: &Arc, + request_headers: &HeaderMap, + index: usize, +) -> Response { + let Some(video) = hls_provisioning_segment_buffer(app_state, index) else { + return StatusCode::FORBIDDEN.into_response(); + }; + build_hls_cvs_response_from_buffer(&video, request_headers.get(header::RANGE)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HlsCvsRange { + Full, + Partial { start: usize, end: usize }, + Unsatisfiable, +} + +fn insert_static_header(headers: &mut HeaderMap, name: HeaderName, value: &'static str) { + headers.insert(name, HeaderValue::from_static(value)); +} + +fn insert_usize_header(headers: &mut HeaderMap, name: HeaderName, value: usize) -> bool { + match HeaderValue::from_str(&value.to_string()) { + Ok(header_value) => { + headers.insert(name, header_value); + true + } + Err(_) => false, + } +} + +fn insert_string_header(headers: &mut HeaderMap, name: HeaderName, value: &str) -> bool { + match HeaderValue::from_str(value) { + Ok(header_value) => { + headers.insert(name, header_value); + true + } + Err(_) => false, + } +} + +fn resolve_hls_cvs_range(range_header: Option<&HeaderValue>, full_size: usize) -> HlsCvsRange { + let Some(range_header) = range_header.and_then(|value| value.to_str().ok()) else { + return HlsCvsRange::Full; + }; + let Some(range) = range_header.strip_prefix("bytes=") else { + return HlsCvsRange::Unsatisfiable; + }; + if full_size == 0 || range.contains(',') { + return HlsCvsRange::Unsatisfiable; + } + let Some((start_raw, end_raw)) = range.split_once('-') else { + return HlsCvsRange::Unsatisfiable; + }; + if start_raw.is_empty() { + let Ok(suffix_len) = end_raw.parse::() else { + return HlsCvsRange::Unsatisfiable; + }; + if suffix_len == 0 { + return HlsCvsRange::Unsatisfiable; + } + let start = full_size.saturating_sub(suffix_len); + return HlsCvsRange::Partial { start, end: full_size - 1 }; + } + let Ok(start) = start_raw.parse::() else { + return HlsCvsRange::Unsatisfiable; + }; + if start >= full_size { + return HlsCvsRange::Unsatisfiable; + } + let end = if end_raw.is_empty() { + full_size - 1 + } else { + let Ok(end) = end_raw.parse::() else { + return HlsCvsRange::Unsatisfiable; + }; + if end < start { + return HlsCvsRange::Unsatisfiable; + } + end.min(full_size - 1) + }; + HlsCvsRange::Partial { start, end } +} + +fn build_hls_cvs_response_from_buffer(video: &TransportStreamBuffer, range_header: Option<&HeaderValue>) -> Response { + // `clone_bytes` returns a `Bytes` (refcount bump) instead of forcing + // `Bytes::copy_from_slice` to memcpy the entire TS payload each response. + let bytes_owned = video.clone_bytes(); + let full_size = bytes_owned.len(); + let range = resolve_hls_cvs_range(range_header, full_size); + let mut response = match range { + HlsCvsRange::Full => { + let mut builder = Response::builder().status(StatusCode::OK); + let headers = builder.headers_mut().expect("response builder headers should be available"); + insert_static_header(headers, header::CONTENT_TYPE, HLS_CVS_CONTENT_TYPE); + insert_static_header(headers, header::ACCEPT_RANGES, ACCEPT_RANGES_VALUE); + insert_static_header(headers, header::CACHE_CONTROL, HLS_CVS_CACHE_CONTROL); + if !insert_usize_header(headers, header::CONTENT_LENGTH, full_size) { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + builder + .body(Body::from(bytes_owned)) + .map_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), IntoResponse::into_response) + } + HlsCvsRange::Partial { start, end } => { + let mut builder = Response::builder().status(StatusCode::PARTIAL_CONTENT); + let headers = builder.headers_mut().expect("response builder headers should be available"); + insert_static_header(headers, header::CONTENT_TYPE, HLS_CVS_CONTENT_TYPE); + insert_static_header(headers, header::ACCEPT_RANGES, ACCEPT_RANGES_VALUE); + insert_static_header(headers, header::CACHE_CONTROL, HLS_CVS_CACHE_CONTROL); + if !insert_usize_header(headers, header::CONTENT_LENGTH, end - start + 1) + || !insert_string_header(headers, header::CONTENT_RANGE, &format!("bytes {start}-{end}/{full_size}")) + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + // `Bytes::slice` shares the underlying allocation (no copy). + builder + .body(Body::from(bytes_owned.slice(start..=end))) + .map_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), IntoResponse::into_response) + } + HlsCvsRange::Unsatisfiable => { + let mut builder = Response::builder().status(StatusCode::RANGE_NOT_SATISFIABLE); + let headers = builder.headers_mut().expect("response builder headers should be available"); + insert_static_header(headers, header::ACCEPT_RANGES, ACCEPT_RANGES_VALUE); + insert_static_header(headers, header::CACHE_CONTROL, HLS_CVS_CACHE_CONTROL); + insert_static_header(headers, header::CONTENT_LENGTH, "0"); + if !insert_string_header(headers, header::CONTENT_RANGE, &format!("bytes */{full_size}")) { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + builder + .body(Body::empty()) + .map_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), IntoResponse::into_response) + } + }; + mark_response_as_uncompressed(&mut response); + response +} + +fn hls_custom_video_buffer( + app_state: &Arc, + custom_video_type: CustomVideoStreamType, +) -> Option { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + let custom_stream_response = custom_stream_response.as_ref()?; + match custom_video_type { + CustomVideoStreamType::ChannelUnavailable => custom_stream_response.channel_unavailable.clone(), + CustomVideoStreamType::UserConnectionsExhausted => custom_stream_response.user_connections_exhausted.clone(), + CustomVideoStreamType::ProviderConnectionsExhausted => custom_stream_response.provider_connections_exhausted.clone(), + CustomVideoStreamType::LowPriorityPreempted => custom_stream_response.low_priority_preempted.clone(), + CustomVideoStreamType::UserAccountExpired => custom_stream_response.user_account_expired.clone(), + CustomVideoStreamType::Provisioning => custom_stream_response.panel_api_provisioning.clone(), + CustomVideoStreamType::HlsSessionOrLeaseExpired => { + custom_stream_response.hls_session_or_lease_expired.clone() + } + } +} + +fn create_hls_custom_video_segment_response( + app_state: &Arc, + request_headers: &HeaderMap, + custom_video_type: CustomVideoStreamType, +) -> Response { + let Some(video) = hls_custom_video_buffer(app_state, custom_video_type) else { + return StatusCode::FORBIDDEN.into_response(); + }; + build_hls_cvs_response_from_buffer(&video, request_headers.get(header::RANGE)) +} + +async fn cvs_provisioning_manifest_api( + fingerprint: Fingerprint, + axum::extract::Path((username, password)): axum::extract::Path<(String, String)>, + axum::extract::Query(query): axum::extract::Query, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let (user, target) = match resolve_cvs_user_context(&app_state, &fingerprint, &username, &password) { + Ok(context) => context, + Err(response) => return *response, }; - if let Err(e) = resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), &app_state) { - return e.into_player_response(app_state.app_config.get_auth_error_status()); - } + let Some(input) = resolve_hls_virtual_input_for_target(&app_state, &target, query.id).await else { + return StatusCode::NOT_FOUND.into_response(); + }; - create_custom_video_stream_response(&app_state, &fingerprint.addr, custom_video_type).into_response() + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, query.id); + let server_path = app_state.app_config.get_user_server_info(&user).and_then(|server| server.path); + hls_panel_provisioning_poll_manifest_response( + &app_state, + &fingerprint, + &user, + &target, + &input, + query.id, + &original_hls_entry_path, + server_path.as_deref(), + ) + .await } pub fn cvs_api_register() -> axum::Router> { - axum::Router::new().route("/cvs/{username}/{password}/{stream_type}", axum::routing::get(cvs_api)) + axum::Router::new() + .route( + "/cvs/hls/{username}/{password}/provisioning.m3u8", + axum::routing::get(cvs_provisioning_manifest_api), + ) + .route("/cvs/{route_kind}/{username}/{password}/{stream_type}", axum::routing::get(cvs_typed_api)) + .route("/cvs/{username}/{password}/{stream_type}", axum::routing::get(cvs_api)) +} + +#[cfg(test)] +mod hls_cvs_tests { + use super::{ + build_hls_cvs_response_from_buffer, resolve_hls_cvs_range, strip_hls_custom_video_media_extension, + HlsCvsRange, + }; + use crate::api::model::TransportStreamBuffer; + use axum::{ + body::to_bytes, + http::{ + header::{self, HeaderValue}, + StatusCode, + }, + }; + + fn test_ts_bytes() -> Vec { + let mut bytes = vec![0_u8; 188 * 2]; + bytes[0] = 0x47; + bytes[188] = 0x47; + bytes[1] = b'a'; + bytes[189] = b'b'; + bytes + } + + fn test_buffer() -> TransportStreamBuffer { TransportStreamBuffer::new(test_ts_bytes()) } + + async fn response_body(response: axum::response::Response) -> bytes::Bytes { + to_bytes(response.into_body(), usize::MAX).await.expect("body should collect") + } + + #[test] + fn hls_cvs_range_zero_open_resolves_to_full_partial_range() { + let range = HeaderValue::from_static("bytes=0-"); + + assert_eq!( + resolve_hls_cvs_range(Some(&range), 376), + HlsCvsRange::Partial { start: 0, end: 375 } + ); + } + + #[test] + fn hls_cvs_suffix_range_resolves_from_tail() { + let range = HeaderValue::from_static("bytes=-10"); + + assert_eq!( + resolve_hls_cvs_range(Some(&range), 376), + HlsCvsRange::Partial { start: 366, end: 375 } + ); + } + + #[test] + fn hls_cvs_multi_range_is_unsatisfiable() { + let range = HeaderValue::from_static("bytes=0-1,4-5"); + + assert_eq!(resolve_hls_cvs_range(Some(&range), 376), HlsCvsRange::Unsatisfiable); + } + + #[test] + fn hls_cvs_media_extension_parser_accepts_supported_segment_types() { + for extension in ["ts", "mp4", "m4s", "m4v"] { + assert_eq!( + strip_hls_custom_video_media_extension(&format!("channel_unavailable.{extension}")), + "channel_unavailable" + ); + } + assert_eq!(strip_hls_custom_video_media_extension("channel_unavailable.m4a"), "channel_unavailable.m4a"); + assert_eq!(strip_hls_custom_video_media_extension("channel_unavailable.m3u8"), "channel_unavailable.m3u8"); + } + + #[tokio::test] + async fn hls_cvs_segment_without_range_returns_finite_ts_body() { + let buffer = test_buffer(); + + let response = build_hls_cvs_response_from_buffer(&buffer, None); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "376"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(!response.headers().contains_key(header::TRANSFER_ENCODING)); + assert_eq!(response_body(response).await, bytes::Bytes::from(test_ts_bytes())); + } + + #[tokio::test] + async fn hls_cvs_segment_with_range_zero_open_returns_partial_content() { + let buffer = test_buffer(); + let range = HeaderValue::from_static("bytes=0-"); + + let response = build_hls_cvs_response_from_buffer(&buffer, Some(&range)); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-375/376"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "376"); + assert_eq!(response_body(response).await, bytes::Bytes::from(test_ts_bytes())); + } + + #[tokio::test] + async fn hls_cvs_segment_with_invalid_range_returns_416() { + let buffer = test_buffer(); + let range = HeaderValue::from_static("bytes=999-"); + + let response = build_hls_cvs_response_from_buffer(&buffer, Some(&range)); + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */376"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(response_body(response).await, bytes::Bytes::new()); + } } #[cfg(test)] @@ -55,21 +555,36 @@ mod tests { use crate::{ api::model::{ ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, DownloadQueue, - EventManager, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, TransportStreamBuffer, - UpdateGuard, + EventManager, HlsProvisioningState, HlsProxyManager, MetadataUpdateManager, PlaylistStorageState, + SharedStreamManager, TransportStreamBuffer, UpdateGuard, }, model::{ - AppConfig, Config, ConfigInput, CustomStreamResponse, MediaToolCapabilities, SourcesConfig, + ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigSource, ConfigTarget, + CustomStreamResponse, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig, TargetOutput, TargetUser, + XtreamTargetFlagsSet, XtreamTargetOutput, }, }; use arc_swap::{ArcSwap, ArcSwapOption}; - use axum::{body::Body, http::{Request, StatusCode}, Router}; + use axum::{ + body::Body, + http::{header, Request, StatusCode}, + response::IntoResponse, + Router, + }; use crate::utils::{FileLockManager, GeoIp}; use std::{collections::HashMap, sync::Arc}; use tower::ServiceExt; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; - use shared::model::{ConfigPaths, InputFetchMethod, InputType}; + use shared::{foundation::Filter, model::{ConfigPaths, InputFetchMethod, InputType, ProcessingOrder}}; + + fn test_fingerprint() -> crate::auth::Fingerprint { + crate::auth::Fingerprint::new( + "test-fingerprint".to_string(), + "127.0.0.1".to_string(), + "127.0.0.1:12345".parse().expect("socket addr"), + ) + } fn create_test_app_config_with_channel_unavailable() -> AppConfig { let input = Arc::new(ConfigInput { @@ -87,13 +602,59 @@ mod tests { aliases: None, ..ConfigInput::default() }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + let target = Arc::new(ConfigTarget { + id: 1, + enabled: true, + name: "target".to_string(), + options: None, + sort: None, + filter: Filter::default(), + output: vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::default(), + trakt: None, + filter: None, + })], + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + watch: None, + use_memory_cache: false, + }); + let sources = SourcesConfig { + inputs: vec![Arc::clone(&input)], + sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::clone(&target)] }], + ..SourcesConfig::default() + }; + let mut expired_user = ProxyUserCredentials::default(); + expired_user.username = "viewer".to_string(); + expired_user.password = "secret".to_string(); + expired_user.exp_date = Some(0); + let expired_user = Arc::new(expired_user); + let api_proxy = ApiProxyConfig { + server: vec![ApiProxyServerInfo { + name: "default".to_string(), + protocol: "https".to_string(), + host: "example.test".to_string(), + port: None, + timezone: "UTC".to_string(), + message: String::new(), + path: Some("iptv".to_string()), + }], + user: vec![TargetUser { target: target.name.clone(), credentials: vec![expired_user] }], + ..ApiProxyConfig::default() + }; let app_cfg = AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), + config: Arc::new(ArcSwap::from_pointee(Config { + user_access_control: true, + custom_stream_response_enabled: true, + ..Config::default() + })), sources: Arc::new(ArcSwap::from_pointee(sources)), hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::from_pointee(api_proxy)), file_locks: Arc::new(FileLockManager::default()), paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { home_path: String::new(), @@ -123,6 +684,8 @@ mod tests { low_priority_preempted: None, user_account_expired: None, panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), }))); app_cfg } @@ -153,6 +716,7 @@ mod tests { metadata: CancellationToken::new(), qos_aggregation: CancellationToken::new(), downloads: CancellationToken::new(), + hls_cache: CancellationToken::new(), }; let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); let (manual_update_sender, _) = mpsc::channel::(1); @@ -170,6 +734,8 @@ mod tests { downloads: Arc::new(DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(HlsProxyManager::new()), + hls_provisioning: Arc::new(HlsProvisioningState::new()), active_users, active_provider, connection_manager, @@ -184,19 +750,17 @@ mod tests { } #[tokio::test] - async fn cvs_api_internal_api_proxy_user_returns_bad_request_for_channel_unavailable() { + async fn custom_video_stream_response_returns_ok_for_channel_unavailable() { let app_state = create_test_app_state(); - let router = cvs_api_register().with_state(app_state); - let token = crate::auth::create_access_token(&[0; 32], 30); - let request = Request::builder() - .method("GET") - .uri(format!("/cvs/api_user/api_user/channel_unavailable.ts?token={token}")) - .body(Body::empty()) - .expect("request"); + let response = crate::api::model::create_custom_video_stream_response( + &app_state, + &test_fingerprint().addr, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); - let response = Router::into_service(router).oneshot(request).await.expect("response"); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); } #[tokio::test] @@ -213,4 +777,71 @@ mod tests { assert!(response.status().is_client_error()); } + + #[tokio::test] + async fn hls_cvs_segment_allows_expired_user_custom_response() { + let app_state = create_test_app_state(); + let headers = axum::http::HeaderMap::new(); + let fingerprint = test_fingerprint(); + + for extension in ["ts", "mp4", "m4s", "m4v"] { + let response = super::cvs_api_response(super::CvsApiResponseContext { + fingerprint: &fingerprint, + username: "viewer", + password: "secret", + stream_type: &format!("channel_unavailable.{extension}"), + route_kind: super::CvsRouteKind::Hls, + request_headers: &headers, + raw_query: None, + app_state: &app_state, + }); + + assert_eq!(response.status(), StatusCode::OK, "{extension} custom response should be served"); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + } + } + + #[tokio::test] + async fn hls_cvs_manifest_allows_expired_user_custom_response() { + let app_state = create_test_app_state(); + let headers = axum::http::HeaderMap::new(); + let fingerprint = test_fingerprint(); + + let response = super::cvs_api_response(super::CvsApiResponseContext { + fingerprint: &fingerprint, + username: "viewer", + password: "secret", + stream_type: "channel_unavailable.m3u8", + route_kind: super::CvsRouteKind::Hls, + request_headers: &headers, + raw_query: None, + app_state: &app_state, + }); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body should collect"); + let body = String::from_utf8(body.to_vec()).expect("manifest is utf-8"); + assert!(body.contains("#EXTM3U")); + assert!(body.contains("/cvs/hls/viewer/secret/channel_unavailable.ts")); + } + + #[tokio::test] + async fn hls_cvs_segment_rejects_unknown_credentials() { + let app_state = create_test_app_state(); + let headers = axum::http::HeaderMap::new(); + let fingerprint = test_fingerprint(); + + let response = super::cvs_api_response(super::CvsApiResponseContext { + fingerprint: &fingerprint, + username: "viewer", + password: "wrong", + stream_type: "channel_unavailable.ts", + route_kind: super::CvsRouteKind::Hls, + request_headers: &headers, + raw_query: None, + app_state: &app_state, + }); + + assert!(response.status().is_client_error()); + } } diff --git a/backend/src/api/endpoints/download_api.rs b/backend/src/api/endpoints/download_api.rs index c92b79e08..72f08c169 100644 --- a/backend/src/api/endpoints/download_api.rs +++ b/backend/src/api/endpoints/download_api.rs @@ -2003,6 +2003,8 @@ mod tests { downloads: Arc::new(DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(crate::api::model::HlsProxyManager::new()), + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), active_users, active_provider, connection_manager, diff --git a/backend/src/api/endpoints/extract_accept_header.rs b/backend/src/api/endpoints/extract_accept_header.rs index ef0c99fce..144c5be8d 100644 --- a/backend/src/api/endpoints/extract_accept_header.rs +++ b/backend/src/api/endpoints/extract_accept_header.rs @@ -16,11 +16,15 @@ where parts: &mut Parts, _state: &B, ) -> impl std::future::Future> + Send { - if let Some(accept_type) = parts.headers.get(axum::http::header::ACCEPT) { + let result = if let Some(accept_type) = parts.headers.get(axum::http::header::ACCEPT) { if let Ok(val) = accept_type.to_str() { - return std::future::ready(Ok(ExtractAcceptHeader(Some(val.to_string())))); + Ok(ExtractAcceptHeader(Some(val.to_string()))) + } else { + Ok(ExtractAcceptHeader(None)) } - } - std::future::ready(Ok(ExtractAcceptHeader(None))) + } else { + Ok(ExtractAcceptHeader(None)) + }; + std::future::ready(result) } } diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index d2678c8a2..6f1e8c508 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -1,51 +1,112 @@ +#![allow(clippy::large_futures)] + +// Cross-import of xtream URL helpers is routed through `xtream_url` (a one-way +// re-export module) so this endpoint file no longer depends on a sibling +// endpoint file directly. See `xtream_url`'s module docs for the ARCH-1 +// roadmap that motivates this re-export. +use super::xtream_url::{get_query_path, get_xtream_player_api_stream_url, ApiStreamContext}; use crate::{ api::{ api_utils::{ - admission_failure_response, connection_priority_for_kind, - create_api_proxy_user, create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, get_headers_from_request, - get_hls_session_ttl_secs, get_stream_alternative_url, is_seek_request, local_stream_response, - try_option_bad_request, try_unwrap_body, - HeaderFilter, + connection_priority_for_kind, create_api_proxy_user, create_playback_session_fingerprint, + create_session_fingerprint, force_provider_stream_response, get_headers_from_request, + get_hls_session_ttl_secs, get_stream_alternative_url, is_hls_stream_share_enabled, is_seek_request, + local_stream_response, record_connect_failed_attempt, resolve_playback_request_admission, + try_option_bad_request, try_unwrap_body, ConnectFailedAttempt, EvictionReentryGuard, HeaderFilter, }, model::{ - AppState, CustomVideoStreamType, ProviderAllocation, UserSession, + begin_hls_origin_account_io_bounded, build_hls_origin_session_owner, build_proxy_session_id, + cold_start_retry_after_seconds, hls_cached_manifest_options_for_requirement, + extract_hls_provider_session_headers, hls_committed_manifest_body_for_request, + hls_custom_video_manifest_path, + hls_custom_video_manifest_response_with_virtual_id, hls_object_body_deadline, hls_origin_account_status, + hls_provisioning_discontinuity_sequence, is_custom_video_stream_enabled, + is_hls_provisioning_gap_segment, is_hls_provisioning_segment, + hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, + hls_transient_object_fetch_failure, hls_transient_origin_response, + hls_virtual_entry_redirect_response, maybe_trigger_origin_refresh, new_hls_access_lease_id, + origin_account_binding_from_allocation, retry_after_secs_from_ms, safe_hls_access_lease_id, + safe_proxy_session_id, safe_user_session_token, scrub_hls_origin_headers, serve_hls_map_cache_outcome, + serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, + serve_hls_transient_object_cache_response, should_remove_hls_origin_header, + start_hls_panel_provisioning_once, + try_hls_panel_provisioning_manifest_response, validate_hls_access_lease, AppState, CacheAccessState, + CustomVideoStreamType, HlsAccessAdmissionMode, + HlsAccessContext, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, + HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, + HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAccountBindingProtection, + HlsCacheResponseContext, HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, HlsCachedManifestOptions, + HlsCommittedManifestBody, HlsEffectiveOriginAcquirePolicy, HlsManifestCommitRequirement, HlsMapFile, + HlsMediaActivityMarker, + HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, + HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginSource, + HlsOriginSourceKind, HlsOriginWorkClass, HlsPanelProvisioningRedirectPaths, HlsPlaybackFamilyKey, + HlsProvisioningStatus, HlsQosMeterInit, HlsQosRuntimeConfig, + HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, + HLS_PROVISIONING_TARGET_DURATION_SECS, + HlsResourceServeFailure, HlsResourceServeOutcome, HlsSegmentFailureObject, + HlsSegmentFailureTransition, HlsSegmentFile, + HlsSession, HlsSessionHandle, HlsSessionKey, + HlsSessionMode, HlsSessionStoreOutcome, LiveHlsOriginEntry, OriginRefreshRequest, OriginSegmentKey, + ProviderAllocation, + ProviderConfig as RuntimeProviderConfig, ProviderHandle, ProxySessionId, RetryPolicy, SegmentCacheKey, + SegmentCacheStatus, SegmentDemandFetchOutcome, SegmentFetchContext, SegmentFetchPolicy, TransientObjectCacheKey, + TransientObjectUnavailableState, TransientResourceFile, TransientResourceRef, + TransientResourceKind, TransportStreamBuffer, UserSession, ConnectionHistoryMode, StreamMeterHandle, + HLS_ACCESS_LEASE_ID_PLACEHOLDER, HlsTransientObjectFetchFailure, HlsTransientOriginIoGuard, + HlsTransientCacheCommitContext, HlsTransientObjectCacheAction, HlsTransientObjectFetchFinalizer, + HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, resolve_hls_transient_object_cache_action, }, + panel_api::can_provision_on_exhausted, }, - auth::{create_access_token, Fingerprint}, - model::{ConfigInput, ConfigInputFlags, ConfigTarget, InputSource, ProxyUserCredentials}, - processing::parser::hls::{get_hls_session_token_and_url_from_token, rewrite_hls, RewriteHlsProps}, - repository::{m3u_get_item_for_stream_id, xtream_get_item_for_stream_id}, + auth::{check_network_access_only, Fingerprint}, + model::{ + ConfigInput, ConfigInputFlags, ConfigProvider, ConfigTarget, InputSource, ProxyUserCredentials, + ReverseProxyDisabledHeaderConfig, + }, + processing::parser::hls::{ + get_hls_session_token_and_url_from_token, rewrite_hls, + initial_strip::{materialize_initial_hls_strip_view, HlsInitialStripOutcome}, + RewriteHlsProps, + }, + repository::{m3u_get_item_for_stream_id, storage_const, xtream_get_item_for_stream_id}, utils::{debug_if_enabled, request, request::is_file_url}, }; -use axum::{http::HeaderMap, response::IntoResponse}; -use log::{debug, error}; -use serde::Deserialize; -use shared::{ - model::{PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, XtreamCluster}, - utils::{is_hls_url, replace_url_extension, sanitize_sensitive_info, Internable, CUSTOM_VIDEO_PREFIX, HLS_EXT}, +use axum::{ + body::Body, + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, }; -use std::{collections::HashMap, sync::Arc}; +use futures::FutureExt; +use log::{debug, error, warn}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use shared::{ + model::{ + ConnectFailureReason, FailureStage, InputType, PlaylistItemType, StreamChannel, StreamInfo, TargetType, + UserConnectionPermission, XtreamCluster, + }, + utils::{ + generate_random_string, is_hls_url, replace_url_extension, sanitize_sensitive_info, Internable, + PROVIDER_SCHEME_PREFIX, + }, + defaults::{HLS_EXT,}, +}; +use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; use url::Url; -use shared::model::ConnectFailureReason; -use crate::auth::check_network_access_only; -const PLAYLIST_TEMPLATE: &str = r"#EXTM3U -#EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:10 -#EXT-X-MEDIA-SEQUENCE:0 -#EXTINF:10.0, -{url} -#EXT-X-ENDLIST -"; const MAX_MANUAL_REDIRECTS: usize = 10; +const HLS_TEMPORARY_RESOURCE_RETRY_AFTER_SECS: u64 = 1; +const HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS: u64 = HLS_TEMPORARY_RESOURCE_RETRY_AFTER_SECS * 1_000; fn is_m3u_catchup_session_token(session_token: &str) -> bool { session_token.starts_with("m3u-catchup|") || session_token.starts_with("catchup|") } -fn query_flag_is_archive(key: &str) -> bool { - key.eq_ignore_ascii_case("utc") -} +fn query_flag_is_archive(key: &str) -> bool { key.eq_ignore_ascii_case("utc") } fn query_flag_marks_start_context(key: &str) -> bool { key.eq_ignore_ascii_case("end") || key.eq_ignore_ascii_case("duration") || key.eq_ignore_ascii_case("lutc") @@ -93,54 +154,1891 @@ struct HlsApiPathParams { token: String, } +#[derive(Debug, Deserialize)] +struct HlsProxySegmentPathParams { + proxy_session_id: String, + hls_access_lease_id: String, + segment_file: String, +} + +#[derive(Debug, Deserialize)] +struct HlsProxyManifestPathParams { + proxy_session_id: String, + hls_access_lease_id: String, +} + +#[derive(Debug, Deserialize)] +struct HlsProxyMapPathParams { + proxy_session_id: String, + hls_access_lease_id: String, + map_file: String, +} + +#[derive(Debug, Deserialize)] +struct HlsProxyResourcePathParams { + proxy_session_id: String, + hls_access_lease_id: String, + resource_file: String, +} + fn hls_response(hls_content: String) -> impl IntoResponse + Send { try_unwrap_body!(axum::response::Response::builder() .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, "application/x-mpegurl") + .header(axum::http::header::CONTENT_TYPE, "application/vnd.apple.mpegurl") + .header(axum::http::header::CACHE_CONTROL, "no-store, no-cache, must-revalidate",) .body(hls_content)) } -fn extract_hls_provider_session_headers(headers: &HeaderMap) -> HashMap { - let cookies = headers - .get_all(axum::http::header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .filter_map(|value| value.split(';').next().map(str::trim)) - .filter(|value| !value.is_empty()) - .collect::>(); - - let mut session_headers = HashMap::new(); - if !cookies.is_empty() { - session_headers.insert(String::from("cookie"), cookies.join("; ")); +fn hls_custom_video_type_for_failure_reason(reason: ConnectFailureReason) -> CustomVideoStreamType { + match reason { + ConnectFailureReason::UserAccountExpired => CustomVideoStreamType::UserAccountExpired, + ConnectFailureReason::UserConnectionsExhausted => CustomVideoStreamType::UserConnectionsExhausted, + ConnectFailureReason::ProviderConnectionsExhausted => CustomVideoStreamType::ProviderConnectionsExhausted, + ConnectFailureReason::Preempted => CustomVideoStreamType::LowPriorityPreempted, + ConnectFailureReason::Provisioning => CustomVideoStreamType::Provisioning, + ConnectFailureReason::ProviderError + | ConnectFailureReason::ProviderClosed + | ConnectFailureReason::ChannelUnavailable + | ConnectFailureReason::SessionExpired => CustomVideoStreamType::ChannelUnavailable, } - session_headers } +pub(crate) fn hls_custom_video_manifest_response( + app_state: &Arc, + user: &ProxyUserCredentials, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + hls_custom_video_manifest_response_with_virtual_id(app_state, user, video_type, fallback_status, None) +} + +pub(crate) fn hls_admission_failure_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + stream_channel: StreamChannel, + provider_name: Arc, + req_headers: &HeaderMap, + reason: ConnectFailureReason, +) -> axum::response::Response { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name, + req_headers, + reason, + failure_stage: FailureStage::Admission, + }); + hls_custom_video_manifest_response( + app_state, + user, + hls_custom_video_type_for_failure_reason(reason), + StatusCode::FORBIDDEN, + ) +} + +fn apply_hls_proxy_public_path_prefix(hls_content: String, server_path: Option<&str>) -> String { + let Some(path_prefix) = normalize_hls_proxy_public_path_prefix(server_path) else { + return hls_content; + }; + + let uri_attr_prefix = format!("URI=\"{path_prefix}/hls/shared/live/"); + let hls_content = hls_content.replace("URI=\"/hls/shared/live/", &uri_attr_prefix); + if hls_content.is_empty() { + return hls_content; + } + let mut prefixed = String::with_capacity(hls_content.len().saturating_add(path_prefix.len().saturating_mul(4))); + + for part in hls_content.split_inclusive('\n') { + let (line, line_ending) = split_hls_line_ending(part); + if line.starts_with("/hls/shared/live/") { + prefixed.push_str(&path_prefix); + } + prefixed.push_str(line); + prefixed.push_str(line_ending); + } + + prefixed +} + +fn normalize_hls_proxy_public_path_prefix(server_path: Option<&str>) -> Option { + let path = server_path?.trim().trim_matches('/'); + if path.is_empty() { + return None; + } + Some(format!("/{path}")) +} + +fn split_hls_line_ending(part: &str) -> (&str, &str) { + if let Some(line) = part.strip_suffix("\r\n") { + (line, "\r\n") + } else if let Some(line) = part.strip_suffix('\n') { + (line, "\n") + } else { + (part, "") + } +} + +fn materialize_hls_access_manifest(hls_content: &str, lease_id: &HlsAccessLeaseId, server_path: Option<&str>) -> String { + let hls_content = hls_content.replace(HLS_ACCESS_LEASE_ID_PLACEHOLDER, &lease_id.0); + apply_hls_proxy_public_path_prefix(hls_content, server_path) +} + +fn hls_access_manifest_uses_startup_view(lease_state: HlsAccessLeaseState) -> bool { + matches!(lease_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle) +} + +#[allow(clippy::too_many_arguments)] +fn materialize_shared_hls_access_manifest( + hls_content: &str, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + mode: &'static str, + server_path: Option<&str>, +) -> String { + let response_body = if hls_access_manifest_uses_startup_view(lease_state) { + let view = materialize_initial_hls_strip_view(hls_content, strip); + match view.outcome { + HlsInitialStripOutcome::Applied { mode: strip_mode, configured, effective, visible_segments } => { + debug!( + "HLS initial strip applied: mode={} lease={} session={} strip_mode={} configured={} effective={} visible_segments={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + strip_mode, + configured, + effective, + visible_segments + ); + } + HlsInitialStripOutcome::Skipped { reason, visible_segments } => { + debug!( + "HLS initial strip skipped: mode={} lease={} session={} reason={} visible_segments={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + reason.as_log_reason(), + visible_segments + ); + } + } + view.body + } else { + debug!( + "HLS initial strip skipped: mode={} lease={} session={} reason={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + if lease_state == HlsAccessLeaseState::Activated { "lease-activated" } else { "lease-not-startup-view" } + ); + hls_content.to_string() + }; + materialize_hls_access_manifest(&response_body, lease_id, server_path) +} + +fn hls_access_lease_ttl_ms(app_state: &Arc) -> u64 { app_state.hls_proxy.session_idle_timeout_ms() } + +fn duration_to_millis_saturating(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +fn hls_pending_bootstrap_window_ms() -> u64 { + duration_to_millis_saturating(hls_initial_manifest_decision_wait_timeout()) +} + +async fn hls_access_lease_timing_for_session( + app_state: &Arc, + session: &HlsSessionHandle, +) -> HlsAccessLeaseTiming { + let timing = session.read().await.account_overlap_timing(); + let active_window_ms = timing.hard_active_window_ms.saturating_mul(2); + HlsAccessLeaseTiming { active_window_ms, valid_window_ms: hls_access_lease_ttl_ms(app_state) } +} + +async fn touch_pending_manifest_follow_up_window( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, +) { + if access_lease_state != HlsAccessLeaseState::Pending { + return; + } + let (proxy_session_id, target_duration) = { + let session = session.read().await; + (session.proxy_session_id.clone(), session.target_duration) + }; + let now_ms = current_time_millis(); + let _ = app_state + .hls_proxy + .mark_pending_manifest_follow_up_for_lease( + access_lease_id, + &proxy_session_id, + now_ms, + target_duration, + ) + .await; +} + +struct HlsResourceAccess { + session: HlsSessionHandle, + access_context: HlsAccessContext, +} + +async fn prepare_hls_resource_access( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + request_kind: &'static str, +) -> Result> { + let Some(session) = app_state.hls_proxy.sessions().get_by_proxy_session_id(proxy_session_id).await else { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + }; + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + proxy_session_id, + now_ms, + ) + .await; + let access_lease_id = HlsAccessLeaseId(hls_access_lease_id.to_string()); + let access_lease_snapshot = + app_state.hls_proxy.access_lease_response_snapshot(&access_lease_id, proxy_session_id, now_ms).await; + let access_context = match validate_hls_proxy_access_request( + app_state, + fingerprint, + proxy_session_id, + hls_access_lease_id, + now_ms, + hls_access_lease_timing_for_session(app_state, &session).await, + request_kind, + ) + .await + { + Ok(context) => context, + Err(err) => { + return Err(Box::new( + hls_resource_access_lease_validation_response( + app_state, + proxy_session_id, + access_lease_snapshot.as_ref(), + now_ms, + err, + ) + .await, + )); + } + }; + let access_lease_snapshot = + app_state.hls_proxy.access_lease_response_snapshot(&access_lease_id, proxy_session_id, now_ms).await; + if let Some(flag) = access_lease_snapshot.as_ref().and_then(|lease| lease.response_flag.as_ref()) { + return Err(Box::new(hls_access_lease_response_flag_resource_response(app_state, &access_context, flag))); + } + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + proxy_session_id, + now_ms, + ) + .await; + reclaim_hls_account_overlap_if_needed(app_state, &session, now_ms).await; + Ok(HlsResourceAccess { session, access_context }) +} + +fn create_hls_cache_user_session_token(fingerprint: &Fingerprint, username: &str, virtual_id: u32) -> String { + let base = create_playback_session_fingerprint(fingerprint, username, virtual_id, PlaylistItemType::LiveHls, None); + format!("{base}|hls-cache|{}", generate_random_string(16)) +} + +fn is_hls_media_activity_status(status: StatusCode) -> bool { + matches!(status, StatusCode::OK | StatusCode::PARTIAL_CONTENT) +} + +async fn hls_cache_response_context( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + now_ms: u64, +) -> HlsCacheResponseContext { + let qos_meter = app_state + .hls_proxy + .qos() + .meter_for_access_lease(&access_context.lease_id) + .await; + HlsCacheResponseContext::new( + access_context.lease_id.clone(), + app_state.hls_proxy.cache_duration_seconds(), + Arc::clone(app_state.hls_proxy.metrics()), + Arc::clone(app_state.hls_proxy.segment_repair()), + qos_meter, + Some(HlsMediaActivityMarker::new(Arc::clone(&app_state.hls_proxy), Arc::clone(session))), + now_ms, + ) +} + +fn hls_qos_meter_init(app_state: &Arc, qos_config: HlsQosRuntimeConfig) -> Option { + if !qos_config.live_metering_enabled { + return None; + } + let meter_uid = app_state.connection_manager.next_stream_uid(); + let meter = Arc::new(StreamMeterHandle::new(meter_uid, Arc::downgrade(&app_state.event_manager))); + Some(HlsQosMeterInit { meter_uid, meter }) +} + +async fn register_hls_cache_stream_for_successful_media_response( + app_state: &Arc, + fingerprint: &Fingerprint, + headers: &HeaderMap, + access_context: &HlsAccessContext, + session: &HlsSessionHandle, + response_context: &HlsCacheResponseContext, +) { + let _ = ensure_hls_cache_stream_registered(app_state, fingerprint, headers, access_context, session).await; + response_context.set_qos_meter(app_state.hls_proxy.qos().meter_for_access_lease(&access_context.lease_id).await); +} + +async fn hls_proxy_segment( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "segment", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + let Some(segment_file) = HlsSegmentFile::parse(¶ms.segment_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + { + let session_guard = session.read().await; + if session_guard.is_gc_marked_for_removal() { + return hls_custom_video_manifest_redirect_response_for_username( + &app_state, + &access_context.username, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + StatusCode::NOT_FOUND, + ); + } + let Some(entry) = session_guard.segments.get(&segment_file.proxy_seq) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + if entry.proxy_file_ext != segment_file.extension { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + } + } + + let preacquired_provider_handle = if hls_segment_request_requires_origin_work(&session, &segment_file).await { + match prepare_hls_origin_binding_for_authorized_resource_work( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + HlsOriginWorkKind::Segment, + now_ms, + ) + .await + { + Ok(handle) => handle, + Err(err) => return hls_origin_runtime_resource_failure_response(&app_state, &access_context, err), + } + } else { + None + }; + + match demand_fetch_hls_segment_if_needed( + &app_state, + &session, + &segment_file, + &access_context, + &fingerprint, + preacquired_provider_handle, + now_ms, + ) + .await + { + SegmentDemandFetchOutcome::NotFound => { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + } + SegmentDemandFetchOutcome::Ready + | SegmentDemandFetchOutcome::QueuedOrFetching + | SegmentDemandFetchOutcome::Unavailable + | SegmentDemandFetchOutcome::TimedOut => {} + } + + let response_context = hls_cache_response_context(&app_state, &session, &access_context, now_ms).await; + let response = hls_resource_serve_outcome_response( + &app_state, + &access_context, + serve_hls_segment_cache_outcome( + Arc::clone(app_state.hls_proxy.segment_cache()), + Arc::clone(&session), + segment_file, + headers.get(header::RANGE).cloned(), + &response_context, + ) + .await, + ); + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + &app_state, + &fingerprint, + &headers, + &access_context, + &session, + &response_context, + ) + .await; + } + response +} + +async fn demand_fetch_hls_segment_if_needed( + app_state: &Arc, + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + preacquired_provider_handle: Option, + now_ms: u64, +) -> SegmentDemandFetchOutcome { + let context = build_hls_segment_fetch_context( + app_state, + session, + Some(access_context.lease_id.clone()), + fingerprint, + preacquired_provider_handle, + ) + .await; + app_state.hls_proxy.segment_worker_pool().demand_fetch_and_wait(context, segment_file, now_ms).await +} + +async fn build_hls_segment_fetch_context( + app_state: &Arc, + session: &HlsSessionHandle, + repair_access_lease_id: Option, + fingerprint: &Fingerprint, + preacquired_provider_handle: Option, +) -> SegmentFetchContext { + let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = { + let session = session.read().await; + ( + session.origin_request_headers.clone(), + session.origin_provider_session_headers.clone(), + session.effective_origin_acquire_policy_or_default(), + session.account_overlap_timing().reservation_ttl_secs(), + ) + }; + let mut origin_io = HlsOriginIoContext { + app_state: Arc::clone(app_state), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::Demand.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + }; + if let Some(provider_handle) = preacquired_provider_handle { + origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + } + SegmentFetchContext { + session: Arc::clone(session), + segment_cache: Arc::clone(app_state.hls_proxy.segment_cache()), + segment_repair: Arc::clone(app_state.hls_proxy.segment_repair()), + repair_access_lease_id, + headers, + origin_provider_session_headers, + client: app_state.http_client.load().as_ref().clone(), + no_redirect_client: app_state.http_client_no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + origin_io: Some(origin_io), + } +} + +async fn hls_effective_origin_acquire_policy(session: &HlsSessionHandle) -> HlsEffectiveOriginAcquirePolicy { + session.read().await.effective_origin_acquire_policy_or_default() +} + +async fn hls_origin_account_reservation_ttl_secs_for_session(session: &HlsSessionHandle) -> u64 { + session.read().await.account_overlap_timing().reservation_ttl_secs() +} + +fn hls_origin_account_reservation_ttl_secs_fallback() -> u64 { + HlsAccountOverlapTiming::from_target_duration_secs(None).reservation_ttl_secs() +} + +async fn hls_proxy_map( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "map", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + let Some(map_file) = HlsMapFile::parse(¶ms.map_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + { + let session_guard = session.read().await; + if session_guard.is_gc_marked_for_removal() { + return hls_custom_video_manifest_redirect_response_for_username( + &app_state, + &access_context.username, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + StatusCode::NOT_FOUND, + ); + } + let Some(entry) = session_guard.maps.get(&map_file.proxy_map_id.into()) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + if entry.proxy_file_ext != map_file.extension { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + } + } + + let response_context = hls_cache_response_context(&app_state, &session, &access_context, now_ms).await; + let response = hls_resource_serve_outcome_response( + &app_state, + &access_context, + serve_hls_map_cache_outcome( + Arc::clone(app_state.hls_proxy.segment_cache()), + Arc::clone(&session), + map_file, + headers.get(header::RANGE).cloned(), + &response_context, + ) + .await, + ); + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + &app_state, + &fingerprint, + &headers, + &access_context, + &session, + &response_context, + ) + .await; + } + response +} + +async fn hls_proxy_resource( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "resource", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + let Some(resource_file) = TransientResourceFile::parse(¶ms.resource_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + let range_header = headers.get(header::RANGE).cloned(); + let cache_duration_ms = app_state.hls_proxy.cache_duration_seconds().saturating_mul(1_000); + let Ok(cache_resolution) = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + &resource_file, + range_header.as_ref(), + now_ms, + cache_duration_ms, + ) + .await + else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + let resource = cache_resolution.resource; + let origin_headers = cache_resolution.origin_headers; + let origin_provider_session_headers = cache_resolution.origin_provider_session_headers; + let cache_action = cache_resolution.action; + + match cache_action { + HlsTransientObjectCacheAction::ServeReady => { + return serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { + app_state: &app_state, + session: &session, + fingerprint: &fingerprint, + headers: &headers, + access_context: &access_context, + resource_file, + range_header, + now_ms, + }) + .await; + } + HlsTransientObjectCacheAction::WaitForFetch(notifier) => { + return wait_for_transient_object_cache_fetch(TransientObjectWaitContext { + app_state: &app_state, + session: &session, + fingerprint: &fingerprint, + headers: &headers, + access_context: &access_context, + resource_file, + range_header, + notifier, + }) + .await; + } + HlsTransientObjectCacheAction::FetchAndCache(_) | HlsTransientObjectCacheAction::PassthroughNoCache => {} + } + + fetch_or_passthrough_transient_resource( + &app_state, + &session, + &fingerprint, + &headers, + &access_context, + &resource, + resource_file, + cache_action, + origin_headers, + origin_provider_session_headers, + range_header, + cache_duration_ms, + now_ms, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn fetch_or_passthrough_transient_resource( + app_state: &Arc, + session: &HlsSessionHandle, + fingerprint: &Fingerprint, + headers: &HeaderMap, + access_context: &HlsAccessContext, + resource: &crate::api::model::TransientResourceRef, + resource_file: TransientResourceFile, + cache_action: HlsTransientObjectCacheAction, + origin_headers: HeaderMap, + origin_provider_session_headers: HeaderMap, + range_header: Option, + cache_duration_ms: u64, + now_ms: u64, +) -> axum::response::Response { + if let HlsTransientObjectCacheAction::FetchAndCache(cache_key) = cache_action { + return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext { + app_state, + session, + fingerprint, + headers, + access_context, + resource, + resource_file, + cache_key, + origin_headers, + origin_provider_session_headers, + range_header, + cache_duration_ms, + }) + .await; + } + + let policy = app_state.hls_proxy.segment_fetch_policy(); + // Hoist `proxy_session_id`: it is immutable for the session lifetime, so + // one read-lock before the fetch replaces three after the fetch. The + // `String` inside is cloned cheaply for the Ok-arm call site. + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let fetch_result = fetch_transient_origin_response_with_provider_io(HlsTransientEndpointOriginFetchRequest { + app_state, + session, + access_context, + fingerprint, + headers, + resource, + resource_file: &resource_file, + origin_headers, + origin_provider_session_headers, + range_header, + policy: policy.clone(), + }) + .await; + match fetch_result.result { + Ok((response, transient_origin_guard)) => { + let media_activity_marker = if response.status().is_success() { + record_successful_transient_segment_fetch(session, resource).await; + mark_hls_authorized_media_access(app_state, session, now_ms).await; + let _ = + ensure_hls_cache_stream_registered(app_state, fingerprint, headers, access_context, session).await; + Some(HlsMediaActivityMarker::new(Arc::clone(&app_state.hls_proxy), Arc::clone(session))) + } else { + None + }; + hls_transient_origin_response( + response, + Arc::clone(&resource.access), + transient_origin_guard, + media_activity_marker, + now_ms, + proxy_session_id.0.clone(), + resource_file.resource_id.0.clone(), + resource.kind, + resource.resolved_origin_uri.clone(), + app_state.hls_proxy.segment_fetch_policy().origin_segment_timeout_ms, + ) + } + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { + if let Some(runtime_err) = fetch_result.runtime_prepare_error { + return hls_origin_runtime_resource_failure_response(app_state, access_context, runtime_err); + } + } + match hls_transient_object_fetch_failure(&err) { + HlsTransientObjectFetchFailure::Retryable => { + let failed_at_ms = current_time_millis(); + if let Some(reason) = + record_temporary_transient_segment_fetch_failure(session, resource, &policy, failed_at_ms).await + { + let _ = app_state + .hls_proxy + .mark_access_leases_channel_unavailable_for_session(&proxy_session_id, failed_at_ms, reason) + .await; + hls_resource_channel_unavailable_response(app_state, access_context) + } else { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + } + HlsTransientObjectFetchFailure::Permanent { status } => { + let failed_at_ms = current_time_millis(); + let _ = app_state + .hls_proxy + .mark_access_leases_channel_unavailable_for_session( + &proxy_session_id, + failed_at_ms, + HlsAccessLeaseChannelUnavailableReason::TransientObjectPermanentFailure { status }, + ) + .await; + hls_resource_channel_unavailable_response(app_state, access_context) + } + } + } + } +} + +struct HlsTransientEndpointOriginFetchRequest<'a> { + app_state: &'a Arc, + session: &'a HlsSessionHandle, + access_context: &'a HlsAccessContext, + fingerprint: &'a Fingerprint, + headers: &'a HeaderMap, + resource: &'a TransientResourceRef, + resource_file: &'a TransientResourceFile, + origin_headers: HeaderMap, + origin_provider_session_headers: HeaderMap, + range_header: Option, + policy: SegmentFetchPolicy, +} + +struct HlsTransientOriginFetchResult { + result: Result<(reqwest::Response, Option), HlsOriginResourceFetchError>, + runtime_prepare_error: Option, +} + +async fn fetch_transient_origin_response_with_provider_io( + request: HlsTransientEndpointOriginFetchRequest<'_>, +) -> HlsTransientOriginFetchResult { + let clients = HlsOriginResourceClients { + client: request.app_state.http_client.load().as_ref().clone(), + no_redirect_client: request.app_state.http_client_no_redirect.load().as_ref().clone(), + use_manual_redirects: request.app_state.should_use_manual_redirects(), + }; + let session_log_id = request.session.read().await.proxy_session_id.0.clone(); + let fetch_request = HlsTransientOriginFetchRequest { + resolved_origin_uri: request.resource.resolved_origin_uri.clone(), + origin_headers: request.origin_headers, + origin_provider_session_headers: request.origin_provider_session_headers, + range_header: request.range_header, + resource_file: request.resource_file.clone(), + resource_kind: request.resource.kind, + clients, + policy: request.policy, + session_log_id, + }; + let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); + let app_state_for_prepare = Arc::clone(request.app_state); + let session_for_prepare = Arc::clone(request.session); + let access_context_for_prepare = request.access_context.clone(); + let fingerprint_for_prepare = request.fingerprint.clone(); + let headers_for_prepare = request.headers.clone(); + let runtime_prepare_error_for_prepare = Arc::clone(&runtime_prepare_error); + let result = fetch_hls_transient_origin_response_with_attempt_prepare( + fetch_request, + move |_attempt| { + let app_state = Arc::clone(&app_state_for_prepare); + let session = Arc::clone(&session_for_prepare); + let access_context = access_context_for_prepare.clone(); + let fingerprint = fingerprint_for_prepare.clone(); + let headers = headers_for_prepare.clone(); + let runtime_prepare_error = Arc::clone(&runtime_prepare_error_for_prepare); + async move { + match prepare_hls_transient_origin_io_for_authorized_resource_work( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + current_time_millis(), + ) + .await + { + Ok(guard) => Ok(guard), + Err(err) => { + *runtime_prepare_error.lock().await = Some(err); + Err(HlsOriginResourceFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::Unavailable, + )) + } + } + } + .boxed() + }, + ) + .await; + let runtime_prepare_error = *runtime_prepare_error.lock().await; + HlsTransientOriginFetchResult { result, runtime_prepare_error } +} + +#[allow(clippy::too_many_arguments)] +struct HlsTransientEndpointCacheFetchContext<'a> { + app_state: &'a Arc, + session: &'a HlsSessionHandle, + fingerprint: &'a Fingerprint, + headers: &'a HeaderMap, + access_context: &'a HlsAccessContext, + resource: &'a crate::api::model::TransientResourceRef, + resource_file: TransientResourceFile, + cache_key: TransientObjectCacheKey, + origin_headers: HeaderMap, + origin_provider_session_headers: HeaderMap, + range_header: Option, + cache_duration_ms: u64, +} + +struct TransientObjectWaitContext<'a> { + app_state: &'a Arc, + session: &'a HlsSessionHandle, + fingerprint: &'a Fingerprint, + headers: &'a HeaderMap, + access_context: &'a HlsAccessContext, + resource_file: TransientResourceFile, + range_header: Option, + notifier: Arc, +} + +struct TransientObjectCacheServeContext<'a> { + app_state: &'a Arc, + session: &'a HlsSessionHandle, + fingerprint: &'a Fingerprint, + headers: &'a HeaderMap, + access_context: &'a HlsAccessContext, + resource_file: TransientResourceFile, + range_header: Option, + now_ms: u64, +} + +async fn serve_transient_object_cache_response_and_mark( + context: TransientObjectCacheServeContext<'_>, +) -> axum::response::Response { + let response_context = + hls_cache_response_context(context.app_state, context.session, context.access_context, context.now_ms).await; + let response = hls_resource_serve_outcome_response( + context.app_state, + context.access_context, + serve_hls_transient_object_cache_outcome( + Arc::clone(context.app_state.hls_proxy.segment_cache()), + Arc::clone(context.session), + context.resource_file, + context.range_header, + &response_context, + ) + .await, + ); + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + context.app_state, + context.fingerprint, + context.headers, + context.access_context, + context.session, + &response_context, + ) + .await; + } + response +} + +async fn serve_transient_object_cache_response_and_mark_or_unavailable( + context: TransientObjectCacheServeContext<'_>, +) -> axum::response::Response { + serve_transient_object_cache_response_and_mark(context).await +} + +async fn wait_for_transient_object_cache_fetch(context: TransientObjectWaitContext<'_>) -> axum::response::Response { + let wait_timeout = context.app_state.hls_proxy.segment_fetch_policy().demand_wait_timeout(); + let safe_resource_id = safe_transient_resource_id(&context.resource_file.resource_id); + debug!( + "HLS transient object wait started: resource_id={} lease={} state=inflight", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id) + ); + let wait_result = tokio::time::timeout(wait_timeout, context.notifier.notified()).await; + if wait_result.is_err() { + debug!( + "HLS transient object wait timed out: resource_id={} lease={} state=inflight", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id) + ); + return hls_transient_object_unavailable_response( + context.app_state, + context.session, + &context.resource_file, + current_time_millis(), + context.access_context, + ) + .await; + } + let response = serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { + app_state: context.app_state, + session: context.session, + fingerprint: context.fingerprint, + headers: context.headers, + access_context: context.access_context, + resource_file: context.resource_file, + range_header: context.range_header, + now_ms: current_time_millis(), + }) + .await; + debug!( + "HLS transient object wait completed: resource_id={} lease={} status={}", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id), + response.status() + ); + response +} + +fn safe_transient_resource_id(resource_id: &crate::api::model::TransientResourceId) -> String { + // Truncate at the first char boundary at or before byte 8 to avoid allocating + // a temporary `String` of 8 chars (and a second UTF-8 walk via `len()`). + let full = resource_id.0.as_str(); + let truncate_at = full + .char_indices() + .nth(8) + .map_or(full.len(), |(byte_idx, _)| byte_idx); + if truncate_at == full.len() { + return full.to_owned(); + } + let mut out = String::with_capacity(truncate_at + 3); + out.push_str(&full[..truncate_at]); + out.push_str("..."); + out +} + +async fn validate_hls_proxy_access_request( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + timing: HlsAccessLeaseTiming, + request_kind: &'static str, +) -> Result { + let context = validate_hls_proxy_access_context( + app_state, + fingerprint, + proxy_session_id, + hls_access_lease_id, + now_ms, + HlsAccessAdmissionMode::ResourceAccess, + ) + .await?; + match app_state.hls_proxy.activate_access_lease(&context.lease_id, proxy_session_id, now_ms, timing).await { + HlsAccessLeaseActivation::Activated { .. } => { + debug!( + "HLS access lease accepted: lease={} proxy_session={} session={} request={request_kind}", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + Ok(context) + } + HlsAccessLeaseActivation::Denied => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} request={request_kind} reason=denied", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + Err(HlsAccessLeaseValidationError::AdmissionDenied) + } + HlsAccessLeaseActivation::Expired + | HlsAccessLeaseActivation::UnknownLease + | HlsAccessLeaseActivation::SessionMismatch => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} request={request_kind} reason=expired", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + Err(HlsAccessLeaseValidationError::Expired) + } + } +} + +async fn ensure_hls_cache_stream_registered( + app_state: &Arc, + fingerprint: &Fingerprint, + req_headers: &HeaderMap, + access: &HlsAccessContext, + session: &HlsSessionHandle, +) -> Option { + let (proxy_session_id, origin_source, origin_account_binding) = { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return None; + } + (session.proxy_session_id.clone(), session.origin_source.clone(), session.origin_account_binding.clone()) + }; + let user = app_state.app_config.get_user_credentials(&access.username)?; + let user_session = + app_state.active_users.get_and_update_user_session(&access.username, &access.user_session_token).await?; + let connection_kind = user_session.connection_kind?; + let priority = connection_priority_for_kind(&user, connection_kind); + let mut stream_channel = build_hls_cache_stream_channel(app_state, access, &origin_source, &proxy_session_id).await; + let provider = hls_cache_stats_provider(&origin_source, origin_account_binding.as_ref(), &user_session); + let user_agent = req_headers + .get(header::USER_AGENT) + .map_or_else(|| Cow::Borrowed(""), |value| String::from_utf8_lossy(value.as_bytes())); + + stream_channel.url = Arc::from(hls_cache_stream_stats_url(&proxy_session_id)); + stream_channel.item_type = PlaylistItemType::LiveHls; + stream_channel.cluster = XtreamCluster::try_from(PlaylistItemType::LiveHls).unwrap_or(stream_channel.cluster); + let shared_stream_id = hls_cache_shared_stream_id(&proxy_session_id); + stream_channel.shared = true; + stream_channel.shared_stream_id = Some(shared_stream_id); + stream_channel.shared_joined_existing = Some( + hls_cache_shared_joined_existing(app_state, shared_stream_id, &access.username, &access.user_session_token) + .await, + ); + let qos_config = HlsQosRuntimeConfig::from_app_config(&app_state.app_config); + let qos_registration = app_state + .hls_proxy + .qos() + .ensure_access_lease( + &access.lease_id, + &proxy_session_id, + current_time_millis(), + hls_qos_meter_init(app_state, qos_config), + ) + .await; + if let Some(meter) = qos_registration.register_meter.as_ref() { + app_state.event_manager.register_meter(Arc::clone(meter)).await; + } + let history_mode = if qos_registration.emit_connect_record { + ConnectionHistoryMode::EmitConnect + } else { + ConnectionHistoryMode::RefreshOnly + }; + + app_state + .connection_manager + .update_connection_with_history_mode(crate::api::model::ConnectionParams { + meter_uid: qos_registration.meter_uid, + username: &access.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority, + soft_priority: user.soft_priority, + fingerprint, + provider, + stream_channel: &stream_channel, + user_agent, + session_token: Some(&access.user_session_token), + }, history_mode) + .await +} + +fn hls_cache_stats_provider( + origin_source: &HlsOriginSource, + origin_account_binding: Option<&HlsOriginAccountBinding>, + user_session: &UserSession, +) -> Arc { + origin_account_binding.filter(|binding| binding.is_active()).map_or_else( + || { + if user_session.provider.is_empty() { + Arc::clone(&origin_source.input_name) + } else { + Arc::clone(&user_session.provider) + } + }, + |binding| Arc::clone(&binding.account_name), + ) +} + +async fn build_hls_cache_stream_channel( + app_state: &Arc, + access: &HlsAccessContext, + origin_source: &HlsOriginSource, + proxy_session_id: &ProxySessionId, +) -> StreamChannel { + if let Some((_, target)) = app_state.app_config.get_target_for_username(&access.username) { + if let Some(mut channel) = get_stream_channel(app_state, &target, access.virtual_id).await { + channel.url = Arc::from(hls_cache_stream_stats_url(proxy_session_id)); + channel.item_type = PlaylistItemType::LiveHls; + channel.cluster = XtreamCluster::try_from(PlaylistItemType::LiveHls).unwrap_or(channel.cluster); + return channel; + } + return fallback_hls_cache_stream_channel(target.id, access.virtual_id, origin_source, proxy_session_id); + } + + fallback_hls_cache_stream_channel(0, access.virtual_id, origin_source, proxy_session_id) +} + +fn fallback_hls_cache_stream_channel( + target_id: u16, + virtual_id: u32, + origin_source: &HlsOriginSource, + proxy_session_id: &ProxySessionId, +) -> StreamChannel { + let unknown = "Unknown".intern(); + StreamChannel { + target_id, + virtual_id, + provider_id: 0, + input_name: Arc::clone(&origin_source.input_name), + item_type: PlaylistItemType::LiveHls, + cluster: XtreamCluster::Live, + group: unknown.clone(), + title: unknown, + url: Arc::from(hls_cache_stream_stats_url(proxy_session_id)), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + } +} + +fn hls_cache_stream_stats_url(proxy_session_id: &ProxySessionId) -> String { + format!("/hls/shared/live/{}/manifest.m3u8", proxy_session_id.0) +} + +fn hls_cache_shared_stream_id(proxy_session_id: &ProxySessionId) -> u64 { + let digest = Sha256::digest(proxy_session_id.0.as_bytes()); + digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)) +} + +async fn hls_cache_shared_joined_existing( + app_state: &Arc, + shared_stream_id: u64, + username: &str, + session_token: &str, +) -> bool { + let streams = app_state.active_users.active_streams().await; + if let Some(existing) = streams.iter().find(|stream| { + stream.username == username + && stream.session_token.as_deref() == Some(session_token) + && stream.channel.shared + && stream.channel.shared_stream_id == Some(shared_stream_id) + }) { + return existing.channel.shared_joined_existing.unwrap_or(false); + } + + streams.iter().any(|stream| { + stream.channel.shared + && stream.channel.shared_stream_id == Some(shared_stream_id) + && (stream.username != username || stream.session_token.as_deref() != Some(session_token)) + }) +} + +async fn validate_hls_proxy_access_context( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + admission_mode: HlsAccessAdmissionMode, +) -> Result { + validate_hls_access_lease( + app_state, + fingerprint, + proxy_session_id, + &HlsAccessLeaseId(hls_access_lease_id.to_string()), + now_ms, + admission_mode, + ) + .await +} + +fn hls_custom_video_manifest_response_for_username( + app_state: &Arc, + username: &str, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + if let Some(user) = app_state.app_config.get_user_credentials(username) { + return hls_custom_video_manifest_response(app_state, &user, video_type, fallback_status); + } + fallback_status.into_response() +} + +fn hls_custom_video_manifest_redirect_response_for_username( + app_state: &Arc, + username: &str, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + if !is_custom_video_stream_enabled(&app_state.app_config) { + return fallback_status.into_response(); + } + let Some(user) = app_state.app_config.get_user_credentials(username) else { + return fallback_status.into_response(); + }; + let server_path = app_state.app_config.get_user_server_info(&user).and_then(|server| server.path); + hls_virtual_entry_redirect_response(&hls_custom_video_manifest_path(&user, video_type), server_path.as_deref()) +} + +fn hls_resource_channel_unavailable_response( + app_state: &Arc, + access_context: &HlsAccessContext, +) -> axum::response::Response { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + &access_context.username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) +} + +fn hls_access_lease_response_flag_manifest_response( + app_state: &Arc, + username: &str, + flag: &HlsAccessLeaseResponseFlag, +) -> axum::response::Response { + match flag { + HlsAccessLeaseResponseFlag::ChannelUnavailable { .. } => hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ), + } +} + +fn hls_manifest_channel_unavailable_response_for_username( + app_state: &Arc, + username: &str, +) -> axum::response::Response { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) +} + +fn hls_manifest_low_priority_preempted_redirect_response_for_username( + app_state: &Arc, + username: &str, +) -> axum::response::Response { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::LowPriorityPreempted, + StatusCode::SERVICE_UNAVAILABLE, + ) +} + +async fn hls_post_refresh_access_lease_response_flag_manifest_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + username: &str, +) -> Option { + let lease = app_state + .hls_proxy + .access_lease_response_snapshot(access_lease_id, proxy_session_id, current_time_millis()) + .await?; + let flag = lease.response_flag?; + Some(hls_access_lease_response_flag_manifest_response(app_state, username, &flag)) +} + +fn hls_access_lease_response_flag_resource_response( + app_state: &Arc, + access_context: &HlsAccessContext, + flag: &HlsAccessLeaseResponseFlag, +) -> axum::response::Response { + match flag { + HlsAccessLeaseResponseFlag::ChannelUnavailable { .. } => { + hls_resource_channel_unavailable_response(app_state, access_context) + } + } +} + +fn hls_origin_runtime_resource_failure_response( + app_state: &Arc, + access_context: &HlsAccessContext, + err: HlsOriginRuntimeAcquireError, +) -> axum::response::Response { + match err { + HlsOriginRuntimeAcquireError::NoAccountAvailable { reason } => { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + &access_context.username, + reason.custom_video_type(), + StatusCode::SERVICE_UNAVAILABLE, + ) + } + HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status), + } +} + +fn hls_resource_serve_outcome_response( + app_state: &Arc, + access_context: &HlsAccessContext, + outcome: HlsResourceServeOutcome, +) -> axum::response::Response { + match outcome { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }) => { + hls_temporary_resource_unavailable_response(retry_after_ms) + } + HlsResourceServeOutcome::Failure( + HlsResourceServeFailure::Missing + | HlsResourceServeFailure::Expired + | HlsResourceServeFailure::PermanentFailed { .. }, + ) => hls_resource_channel_unavailable_response(app_state, access_context), + } +} + +fn hls_session_or_lease_expired_manifest_response( + app_state: &Arc, + username: &str, +) -> axum::response::Response { + hls_custom_video_manifest_response_for_username( + app_state, + username, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + StatusCode::NOT_FOUND, + ) +} + +async fn hls_manifest_access_lease_validation_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_snapshot: Option<&HlsAccessLease>, + now_ms: u64, + err: HlsAccessLeaseValidationError, +) -> axum::response::Response { + let marker = if lease_snapshot.is_none() { + app_state.hls_proxy.expired_session_marker(proxy_session_id, now_ms).await + } else { + None + }; + let username = + lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); + match err { + HlsAccessLeaseValidationError::AdmissionDenied => username.map_or_else( + || StatusCode::FORBIDDEN.into_response(), + |username| { + hls_custom_video_manifest_response_for_username( + app_state, + username, + CustomVideoStreamType::UserConnectionsExhausted, + StatusCode::FORBIDDEN, + ) + }, + ), + HlsAccessLeaseValidationError::InvalidLease + | HlsAccessLeaseValidationError::SessionMismatch + | HlsAccessLeaseValidationError::UserSessionMissing + | HlsAccessLeaseValidationError::Expired => username.map_or_else( + || StatusCode::NOT_FOUND.into_response(), + |username| hls_session_or_lease_expired_manifest_response(app_state, username), + ), + } +} + +async fn hls_resource_access_lease_validation_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_snapshot: Option<&HlsAccessLease>, + now_ms: u64, + err: HlsAccessLeaseValidationError, +) -> axum::response::Response { + let marker = if lease_snapshot.is_none() { + app_state.hls_proxy.expired_session_marker(proxy_session_id, now_ms).await + } else { + None + }; + let username = + lease_snapshot.map(|lease| lease.username.as_str()).or_else(|| marker.as_ref()?.username.as_deref()); + match err { + HlsAccessLeaseValidationError::AdmissionDenied => username.map_or_else( + || StatusCode::FORBIDDEN.into_response(), + |username| { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::UserConnectionsExhausted, + StatusCode::FORBIDDEN, + ) + }, + ), + HlsAccessLeaseValidationError::InvalidLease + | HlsAccessLeaseValidationError::SessionMismatch + | HlsAccessLeaseValidationError::UserSessionMissing + | HlsAccessLeaseValidationError::Expired => username.map_or_else( + || StatusCode::NOT_FOUND.into_response(), + |username| { + hls_custom_video_manifest_redirect_response_for_username( + app_state, + username, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + StatusCode::NOT_FOUND, + ) + }, + ), + } +} + +async fn hls_manifest_access_context_and_state( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_snapshot: Option<&HlsAccessLease>, + now_ms: u64, +) -> Result<(HlsAccessContext, HlsAccessLeaseState), Box> { + let access_context = match validate_hls_proxy_access_context( + app_state, + fingerprint, + proxy_session_id, + &access_lease_id.0, + now_ms, + HlsAccessAdmissionMode::ManifestPrepare, + ) + .await + { + Ok(context) => context, + Err(err) => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} session= reason={err:?}", + safe_hls_access_lease_id(access_lease_id), + safe_proxy_session_id(proxy_session_id) + ); + return Err(Box::new(hls_manifest_access_lease_validation_response( + app_state, + proxy_session_id, + access_lease_snapshot, + now_ms, + err, + ) + .await)); + } + }; + if access_lease_snapshot.is_none() + && app_state.hls_proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.is_none() + && app_state.hls_proxy.expired_session_marker(proxy_session_id, now_ms).await.is_some() + { + return Err(Box::new(hls_session_or_lease_expired_manifest_response(app_state, &access_context.username))); + } + if let Some(flag) = access_lease_snapshot.and_then(|lease| lease.response_flag.as_ref()) { + return Err(Box::new(hls_access_lease_response_flag_manifest_response( + app_state, + &access_context.username, + flag, + ))); + } + debug!( + "HLS access lease accepted: lease={} proxy_session={} session={} request=manifest", + safe_hls_access_lease_id(&access_context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&access_context.user_session_token) + ); + + let access_lease_state = match app_state + .hls_proxy + .touch_manifest_access_lease( + &access_context.lease_id, + proxy_session_id, + now_ms, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(hls_pending_bootstrap_window_ms()), + }), + hls_access_lease_ttl_ms(app_state), + ) + .await + { + HlsAccessLeaseTouch::Touched { lease } => lease.state, + HlsAccessLeaseTouch::Denied => { + return Err(Box::new(hls_custom_video_manifest_response_for_username( + app_state, + &access_context.username, + CustomVideoStreamType::UserConnectionsExhausted, + StatusCode::FORBIDDEN, + ))); + } + HlsAccessLeaseTouch::Expired | HlsAccessLeaseTouch::UnknownLease | HlsAccessLeaseTouch::SessionMismatch => { + return Err(Box::new(hls_session_or_lease_expired_manifest_response(app_state, &access_context.username))); + } + }; + + Ok((access_context, access_lease_state)) +} + +fn hls_temporary_resource_unavailable_response(retry_after_ms: u64) -> axum::response::Response { + try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header(header::RETRY_AFTER, retry_after_secs_from_ms(retry_after_ms).to_string()) + .body(Body::empty())) +} + +async fn hls_transient_object_unavailable_response( + app_state: &Arc, + session: &HlsSessionHandle, + resource_file: &TransientResourceFile, + now_ms: u64, + access_context: &HlsAccessContext, +) -> axum::response::Response { + let state = { + let session = session.read().await; + let key = crate::api::model::TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &resource_file.resource_id, + resource_file.extension.clone(), + ); + session.transient.object_unavailable_state(&key, now_ms) + }; + match state { + TransientObjectUnavailableState::Fetching => { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + TransientObjectUnavailableState::FailedRetryable { retry_after_ms } => { + hls_temporary_resource_unavailable_response(retry_after_ms) + } + TransientObjectUnavailableState::FailedPermanent | TransientObjectUnavailableState::Missing => { + hls_resource_channel_unavailable_response(app_state, access_context) + } + } +} + +async fn record_successful_transient_segment_fetch(session: &HlsSessionHandle, resource: &crate::api::model::TransientResourceRef) { + if resource.kind != TransientResourceKind::Segment { + return; + } + let mut session = session.write().await; + if let Some(reset_failures) = session.record_successful_segment_fetch() { + debug!( + "HLS segment temporary failure counter reset: session={} previous_failures={reset_failures}", + safe_proxy_session_id(&session.proxy_session_id) + ); + } +} + +async fn record_temporary_transient_segment_fetch_failure( + session: &HlsSessionHandle, + resource: &crate::api::model::TransientResourceRef, + policy: &SegmentFetchPolicy, + now_ms: u64, +) -> Option { + if resource.kind != TransientResourceKind::Segment { + return None; + } + let mut session = session.write().await; + let threshold = session.segment_temporary_failure_threshold(policy.permanent_failure_segment_threshold); + match session.record_temporary_segment_fetch_failure( + now_ms, + HlsSegmentFailureObject::Transient { + resource_id: resource.id.0.clone(), + }, + threshold, + ) { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { + debug!( + "HLS segment temporary failure counted: session={} object={} failures={} threshold={}", + safe_proxy_session_id(&session.proxy_session_id), + resource.id.0, + failures, + threshold + ); + None + } + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { + warn!( + "HLS segment temporary failure threshold reached: session={} failures={} threshold={}", + safe_proxy_session_id(&session.proxy_session_id), + failures, + threshold + ); + session.invalidate_queued_origin_work(); + Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectTemporaryFailureThreshold { + failures, + threshold, + }) + } + } +} + +#[allow(clippy::too_many_lines)] +async fn fetch_and_cache_transient_origin_response( + context: HlsTransientEndpointCacheFetchContext<'_>, +) -> axum::response::Response { + let policy = context.app_state.hls_proxy.segment_fetch_policy(); + let mut fetch_finalizer = HlsTransientObjectFetchFinalizer::new( + Arc::clone(context.session), + context.cache_key.clone(), + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + let clients = HlsOriginResourceClients { + client: context.app_state.http_client.load().as_ref().clone(), + no_redirect_client: context.app_state.http_client_no_redirect.load().as_ref().clone(), + use_manual_redirects: context.app_state.should_use_manual_redirects(), + }; + let session_log_id = context.session.read().await.proxy_session_id.0.clone(); + let fetch_request = HlsTransientOriginFetchRequest { + resolved_origin_uri: context.resource.resolved_origin_uri.clone(), + origin_headers: context.origin_headers.clone(), + origin_provider_session_headers: context.origin_provider_session_headers.clone(), + range_header: context.range_header.clone(), + resource_file: context.resource_file.clone(), + resource_kind: context.resource.kind, + clients, + policy: policy.clone(), + session_log_id, + }; + let cache_fetch_request = HlsTransientOriginCacheFetchRequest { + fetch: fetch_request, + commit: HlsTransientCacheCommitContext { + segment_cache: Arc::clone(context.app_state.hls_proxy.segment_cache()), + segment_repair: Arc::clone(context.app_state.hls_proxy.segment_repair()), + session: Arc::clone(context.session), + access_lease_id: context.access_context.lease_id.clone(), + resource: context.resource.clone(), + resource_file: context.resource_file.clone(), + cache_key: context.cache_key.clone(), + range_header: context.range_header.clone(), + cache_duration_ms: context.cache_duration_ms, + origin_segment_timeout_ms: policy.origin_segment_timeout_ms, + }, + }; + let app_state_for_prepare = Arc::clone(context.app_state); + let session_for_prepare = Arc::clone(context.session); + let access_context = context.access_context.clone(); + let fingerprint_for_prepare = context.fingerprint.clone(); + let headers_for_prepare = context.headers.clone(); + let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); + let runtime_prepare_error_for_prepare = Arc::clone(&runtime_prepare_error); + let final_failure = match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare( + cache_fetch_request, + move |_attempt| { + let app_state = Arc::clone(&app_state_for_prepare); + let session = Arc::clone(&session_for_prepare); + let access_context = access_context.clone(); + let fingerprint = fingerprint_for_prepare.clone(); + let headers = headers_for_prepare.clone(); + let runtime_prepare_error = Arc::clone(&runtime_prepare_error_for_prepare); + async move { + match prepare_hls_transient_origin_io_for_authorized_resource_work( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + current_time_millis(), + ) + .await + { + Ok(guard) => Ok(guard), + Err(err) => { + *runtime_prepare_error.lock().await = Some(err); + Err(HlsOriginResourceFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::Unavailable, + )) + } + } + } + .boxed() + }, + ) + .await + { + Ok(()) => { + let ready_at_ms = current_time_millis(); + let response_context = + hls_cache_response_context(context.app_state, context.session, context.access_context, ready_at_ms) + .await; + let response = serve_hls_transient_object_cache_response( + Arc::clone(context.app_state.hls_proxy.segment_cache()), + Arc::clone(context.session), + context.resource_file.clone(), + context.range_header.clone(), + &response_context, + ) + .await; + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + context.app_state, + context.fingerprint, + context.headers, + context.access_context, + context.session, + &response_context, + ) + .await; + } + record_successful_transient_segment_fetch(context.session, context.resource).await; + fetch_finalizer.complete(); + return response; + } + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { + if let Some(runtime_err) = *runtime_prepare_error.lock().await { + fetch_finalizer.complete(); + context.session.write().await.transient.mark_object_failed_retryable( + &context.cache_key, + current_time_millis(), + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + return hls_origin_runtime_resource_failure_response( + context.app_state, + context.access_context, + runtime_err, + ); + } + } + hls_transient_object_fetch_failure(&err) + } + }; + + let failed_at_ms = current_time_millis(); + let mut response_flag_reason = None; + match final_failure { + HlsTransientObjectFetchFailure::Retryable => { + if let Some(reason) = + record_temporary_transient_segment_fetch_failure(context.session, context.resource, &policy, failed_at_ms) + .await + { + response_flag_reason = Some(reason); + context.session.write().await.transient.mark_object_failed_permanent( + &context.cache_key, + failed_at_ms, + None, + ); + } else { + context.session.write().await.transient.mark_object_failed_retryable( + &context.cache_key, + failed_at_ms, + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + } + } + HlsTransientObjectFetchFailure::Permanent { status } => { + context.session.write().await.transient.mark_object_failed_permanent( + &context.cache_key, + failed_at_ms, + status, + ); + response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::TransientObjectPermanentFailure { + status, + }); + } + } + if let Some(reason) = response_flag_reason { + let proxy_session_id = context.session.read().await.proxy_session_id.clone(); + let marked = context + .app_state + .hls_proxy + .mark_access_leases_channel_unavailable_for_session(&proxy_session_id, failed_at_ms, reason) + .await; + if marked > 0 { + debug!( + "HLS access leases marked channel unavailable: session={} marked={marked}", + safe_proxy_session_id(&proxy_session_id) + ); + } + } + fetch_finalizer.complete(); + hls_transient_object_unavailable_response( + context.app_state, + context.session, + &context.resource_file, + failed_at_ms, + context.access_context, + ) + .await +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + async fn release_prepared_hls_manifest_session( app_state: &Arc, username: &str, session_token: &str, addr: &std::net::SocketAddr, ) { - let _transition_guard = app_state - .active_users - .acquire_playback_transition(username, session_token) - .await; - app_state - .active_users - .release_unbound_session_reservation(username, session_token, None, false) - .await; - app_state - .active_users - .clear_unbound_session_addr(username, session_token, addr) - .await; + let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; + app_state.active_users.release_unbound_session_reservation(username, session_token, None, false).await; + app_state.active_users.clear_unbound_session_addr(username, session_token, addr).await; } async fn terminate_failed_hls_manifest_session(app_state: &Arc, username: &str, session_token: &str) { - let _transition_guard = app_state - .active_users - .acquire_playback_transition(username, session_token) - .await; + let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; app_state.active_users.terminate_session(username, session_token).await; app_state.active_provider.clear_provider_reservation(session_token).await; } @@ -170,25 +2068,3147 @@ fn normalize_xtream_live_hls_url(hls_url: &str, input: &ConfigInput) -> String { hls_url.to_string() } +fn ensure_hls_manifest_extension(url: &str) -> String { + let with_extension = replace_url_extension(url, HLS_EXT); + let (base_url, suffix) = match with_extension.find(['?', '#'].as_ref()) { + Some(pos) => (&with_extension[..pos], &with_extension[pos..]), + None => (with_extension.as_str(), ""), + }; + let Some(path_without_ext) = base_url.strip_suffix(HLS_EXT) else { + return with_extension; + }; + format!("{}{}{}", path_without_ext.trim_end_matches('.'), HLS_EXT, suffix) +} + +fn build_hls_manifest_request_headers( + input_headers: &HashMap, + req_headers: &HeaderMap, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + default_user_agent: Option<&str>, +) -> HeaderMap { + let input_headers = input_headers + .iter() + .filter(|(key, _)| !should_remove_hls_origin_header(key, disabled_headers)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + let disabled_headers_for_filter = disabled_headers.cloned(); + let filter_header: HeaderFilter = Some(Box::new(move |name: &str| { + !name.eq_ignore_ascii_case("range") + && !should_remove_hls_origin_header(name, disabled_headers_for_filter.as_ref()) + })); + let forwarded = get_headers_from_request(req_headers, &filter_header); + let mut headers = + request::get_request_headers(Some(&input_headers), Some(&forwarded), disabled_headers, default_user_agent); + scrub_hls_origin_headers(&mut headers, disabled_headers); + headers +} + +struct HlsCacheManifestOrigin<'a> { + raw_request_url: &'a str, + session_entry_url: HlsOriginEntryUrl, + input: &'a ConfigInput, + origin_source: HlsOriginSource, +} + +struct HlsCacheOriginResolution { + hls_url: String, + session_entry_url: HlsOriginEntryUrl, +} + +#[derive(Clone, Debug)] +enum HlsOriginEntryUrl { + DirectHttp { url: String }, + ProviderFailover { url: String, provider: Arc }, +} + +impl HlsOriginEntryUrl { + fn direct_http(url: impl Into) -> Self { Self::DirectHttp { url: url.into() } } + + fn provider_failover(url: impl Into, provider: Arc) -> Self { + Self::ProviderFailover { url: url.into(), provider } + } + + fn as_str(&self) -> &str { + match self { + Self::DirectHttp { url } | Self::ProviderFailover { url, .. } => url, + } + } + + fn url_failover_provider(&self) -> Option> { + match self { + Self::DirectHttp { .. } => None, + Self::ProviderFailover { provider, .. } => Some(Arc::clone(provider)), + } + } +} + +fn resolve_hls_cache_origin_entry_url(input: &ConfigInput, url: &str) -> Option { + if let Some(provider) = hls_url_failover_provider_for_origin_url(input, url) { + return Some(HlsCacheOriginResolution { + hls_url: url.to_string(), + session_entry_url: HlsOriginEntryUrl::provider_failover(url, provider), + }); + } + + let parsed = Url::parse(url).ok()?; + if matches!(parsed.scheme(), "http" | "https") { + return Some(HlsCacheOriginResolution { + hls_url: url.to_string(), + session_entry_url: HlsOriginEntryUrl::direct_http(url), + }); + } + + warn!("HLS origin entry URL is not supported: url={}", sanitize_sensitive_info(url)); + None +} + +fn hls_url_failover_provider_for_origin_url(input: &ConfigInput, url: &str) -> Option> { + if !url.starts_with(PROVIDER_SCHEME_PREFIX) { + return None; + } + input.get_resolve_provider(url).map(|provider| Arc::clone(&provider)) +} + +fn is_http_hls_origin_url(url: &str) -> bool { + Url::parse(url).is_ok_and(|parsed| matches!(parsed.scheme(), "http" | "https")) +} + +fn is_supported_hls_origin_url(input: &ConfigInput, url: &str) -> bool { + input.get_resolve_provider(url).is_some() || is_http_hls_origin_url(url) +} + +fn hls_stream_ref_from_virtual_id(virtual_id: u32) -> String { virtual_id.to_string() } + +fn build_hls_origin_source(input: &ConfigInput, stream_ref: impl Into) -> HlsOriginSource { + HlsOriginSource::new(input.id, Arc::clone(&input.name), stream_ref, hls_origin_source_kind(input.input_type)) +} + +fn hls_origin_source_kind(input_type: InputType) -> HlsOriginSourceKind { + if input_type.is_xtream() { + HlsOriginSourceKind::XtreamLive + } else if input_type.is_m3u() { + HlsOriginSourceKind::M3uMediaPlaylist + } else { + HlsOriginSourceKind::DirectMediaPlaylist + } +} + +fn build_hls_origin_resolution(input: &ConfigInput, media_playlist_url: &str) -> Option { + let candidate = match hls_origin_source_kind(input.input_type) { + HlsOriginSourceKind::XtreamLive => { + ensure_hls_manifest_extension(&normalize_xtream_live_hls_url(media_playlist_url, input)) + } + HlsOriginSourceKind::M3uMediaPlaylist | HlsOriginSourceKind::DirectMediaPlaylist => { + ensure_hls_manifest_extension(media_playlist_url) + } + }; + resolve_hls_cache_origin_entry_url(input, &candidate) +} + +#[derive(Clone, Copy)] +enum HlsOriginWorkKind { + Manifest, + Segment, + Resource, +} + +impl HlsOriginWorkKind { + const fn as_log_value(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Segment => "segment", + Self::Resource => "resource", + } + } +} + +fn build_hls_origin_fetch_url( + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + provider_config: Option<&Arc>, +) -> Option { + let provider_scheme_url = [session_entry_url, raw_request_url] + .into_iter() + .find(|url| hls_url_failover_provider_for_origin_url(input, url).is_some()); + let url = if let (Some(provider_config), Some(provider_scheme_url)) = (provider_config, provider_scheme_url) { + rewrite_hls_provider_scheme_origin_account(provider_scheme_url, input, provider_config)? + } else if let Some(provider_config) = provider_config { + get_stream_alternative_url(raw_request_url, input, provider_config) + .or_else(|| get_stream_alternative_url(session_entry_url, input, provider_config)) + .unwrap_or_else(|| session_entry_url.to_string()) + } else { + session_entry_url.to_string() + }; + + if is_supported_hls_origin_url(input, &url) { + Some(url) + } else { + None + } +} + +fn rewrite_hls_provider_scheme_origin_account( + provider_scheme_url: &str, + input: &ConfigInput, + provider_config: &Arc, +) -> Option { + if !provider_scheme_url.starts_with(PROVIDER_SCHEME_PREFIX) { + return None; + } + let alt_input_user_info = provider_config.get_user_info()?; + let Some((_source_base_url, source_username, source_password)) = + input.get_matched_config_by_url(provider_scheme_url) + else { + return Some(provider_scheme_url.to_string()); + }; + let (Some(old_username), Some(old_password)) = (source_username, source_password) else { + return Some(provider_scheme_url.to_string()); + }; + + let mut url = Url::parse(provider_scheme_url).ok()?; + if rewrite_hls_url_auth_fields( + &mut url, + old_username, + old_password, + &alt_input_user_info.username, + &alt_input_user_info.password, + ) { + Some(url.to_string()) + } else { + None + } +} + +fn rewrite_hls_url_auth_fields( + url: &mut Url, + old_username: &str, + old_password: &str, + new_username: &str, + new_password: &str, +) -> bool { + if rewrite_hls_query_auth_fields(url, new_username, new_password) { + return true; + } + + if url.username() == old_username && url.password() == Some(old_password) { + return url.set_username(new_username).is_ok() && url.set_password(Some(new_password)).is_ok(); + } + + rewrite_hls_path_auth_fields(url, old_username, old_password, new_username, new_password) +} + +fn rewrite_hls_query_auth_fields(url: &mut Url, new_username: &str, new_password: &str) -> bool { + let mut has_username = false; + let mut has_password = false; + let pairs: Vec<(String, String)> = url + .query_pairs() + .map(|(key, value)| { + if key.eq_ignore_ascii_case("username") { + has_username = true; + (key.into_owned(), new_username.to_string()) + } else if key.eq_ignore_ascii_case("password") { + has_password = true; + (key.into_owned(), new_password.to_string()) + } else { + (key.into_owned(), value.into_owned()) + } + }) + .collect(); + + if !(has_username && has_password) { + return false; + } + + url.query_pairs_mut().clear().extend_pairs(pairs.iter().map(|(key, value)| (key.as_str(), value.as_str()))); + true +} + +fn rewrite_hls_path_auth_fields( + url: &mut Url, + old_username: &str, + old_password: &str, + new_username: &str, + new_password: &str, +) -> bool { + let Some(mut segments) = url + .path_segments() + .map(|segments| segments.map(ToOwned::to_owned).collect::>()) + else { + return false; + }; + + let credential_index = if segments.len() >= 3 + && matches!(segments.first().map(String::as_str), Some("live" | "movie" | "series")) + && segments.get(1).is_some_and(|segment| segment == old_username) + && segments.get(2).is_some_and(|segment| segment == old_password) + { + Some(1) + } else if segments.len() >= 2 + && segments.first().is_some_and(|segment| segment == old_username) + && segments.get(1).is_some_and(|segment| segment == old_password) + { + Some(0) + } else { + None + }; + + let Some(credential_index) = credential_index else { + return false; + }; + + segments[credential_index] = new_username.to_string(); + segments[credential_index + 1] = new_password.to_string(); + + let Ok(mut path_segments) = url.path_segments_mut() else { + return false; + }; + path_segments.clear().extend(segments.iter().map(String::as_str)); + true +} + +fn hls_url_failover_provider_for_origin_context( + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + fetch_url: &str, +) -> Option> { + hls_url_failover_provider_for_origin_url(input, session_entry_url) + .or_else(|| hls_url_failover_provider_for_origin_url(input, raw_request_url)) + .or_else(|| hls_url_failover_provider_for_origin_url(input, fetch_url)) +} + +struct PreparedHlsOriginRuntime { + fetch_url: String, + // URL failover comes from source.yml provider:// resolution. Origin-account + // binding/handles are runtime account reservations and must stay separate. + url_failover_provider: Option>, + origin_account_binding_to_store: Option, + preacquired_origin_account_handle: Option, +} + +fn effective_hls_url_failover_provider_for_fetch_url( + fetch_url: &str, + prepared_url_failover_provider: Option>, + origin_url_failover_provider: Option>, +) -> Option> { + if !fetch_url.starts_with(PROVIDER_SCHEME_PREFIX) { + return None; + } + prepared_url_failover_provider.or(origin_url_failover_provider) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HlsOriginRuntimeAcquireError { + NoAccountAvailable { reason: HlsOriginRuntimeNoAccountReason }, + Fatal(StatusCode), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HlsOriginRuntimeNoAccountReason { + ProviderConnectionsExhausted, + OriginBindingPreempted, +} + +impl HlsOriginRuntimeNoAccountReason { + const fn custom_video_type(self) -> CustomVideoStreamType { + match self { + Self::ProviderConnectionsExhausted => CustomVideoStreamType::ProviderConnectionsExhausted, + Self::OriginBindingPreempted => CustomVideoStreamType::LowPriorityPreempted, + } + } +} + +fn hls_no_account_reason_for_binding( + binding: Option<&HlsOriginAccountBinding>, +) -> HlsOriginRuntimeNoAccountReason { + let Some(binding) = binding else { + return HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted; + }; + match &binding.binding_mode { + HlsOriginAccountBindingMode::Detached { + reason: HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, + .. + } + | HlsOriginAccountBindingMode::Detached { + reason: HlsOriginAccountDetachedReason::PreemptedByHigherPriority, + .. + } => HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, + HlsOriginAccountBindingMode::Detached { .. } + | HlsOriginAccountBindingMode::Active + | HlsOriginAccountBindingMode::Speculative { .. } => { + HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted + } + } +} + +#[derive(Clone)] +struct HlsAccountOverlapCandidate { + proxy_session_id: ProxySessionId, + input_name: Arc, + account_name: Arc, + session_owner: String, + reclaim_until_ms: u64, + last_media_at_ms: u64, + soft_overlap_eligible_at_ms: u64, + soft_overlap_delay_ms: u64, + tuliprox_target_user_connection_capacity: u32, + origin_input_account_connection_capacity: u32, +} + +#[derive(Clone)] +struct HlsOriginPolicyPreemptCandidate { + session: HlsSessionHandle, + proxy_session_id: ProxySessionId, + account_name: Arc, + session_owner: String, + reservation_ttl_secs: u64, + victim_policy: HlsEffectiveOriginAcquirePolicy, + last_media_at_ms: u64, +} + +#[derive(Debug, Clone, Copy)] +struct HlsSoftOverlapCapacity { + tuliprox_target_user_connection_capacity: u32, + origin_input_account_connection_capacity: u32, + delay_ms: u64, +} + +fn hls_soft_overlap_capacity_for_target_duration( + tuliprox_target_user_connection_capacity: u32, + origin_input_account_connection_capacity: u32, + target_duration_ms: u64, +) -> HlsSoftOverlapCapacity { + let delay_ms = hls_soft_overlap_delay_ms( + target_duration_ms, + tuliprox_target_user_connection_capacity, + origin_input_account_connection_capacity, + ); + HlsSoftOverlapCapacity { + tuliprox_target_user_connection_capacity, + origin_input_account_connection_capacity, + delay_ms, + } +} + +async fn hls_origin_input_account_connection_capacity(app_state: &Arc, input: &ConfigInput) -> u32 { + let capacities = app_state.active_provider.provider_capacities_for_input(&input.name).await; + if capacities.is_empty() { + return hls_configured_origin_input_account_connection_capacity(input); + } + capacities + .into_iter() + .map(|(_, _, max)| if max == 0 { u32::MAX } else { u32::try_from(max).unwrap_or(u32::MAX) }) + .fold(0u32, u32::saturating_add) + .max(1) +} + +fn hls_configured_origin_input_account_connection_capacity(input: &ConfigInput) -> u32 { + let input_capacity = if input.max_connections == 0 { 1 } else { u32::from(input.max_connections) }; + input + .aliases + .as_ref() + .map_or(0, |aliases| { + aliases + .iter() + .filter(|alias| alias.enabled) + .map(|alias| if alias.max_connections == 0 { 1 } else { u32::from(alias.max_connections) }) + .fold(0u32, u32::saturating_add) + }) + .saturating_add(input_capacity) + .max(1) +} + +async fn hls_tuliprox_target_user_connection_capacity(app_state: &Arc, input: &ConfigInput) -> u32 { + hls_configured_tuliprox_target_user_connection_capacity(app_state, input) + .max(hls_active_tuliprox_target_user_connections_for_input(app_state, input).await) + .max(1) +} + +fn hls_configured_tuliprox_target_user_connection_capacity(app_state: &Arc, input: &ConfigInput) -> u32 { + let Some(api_proxy) = app_state.app_config.api_proxy.load().as_ref().cloned() else { + return 1; + }; + api_proxy + .user + .iter() + .filter(|target_user| { + app_state + .app_config + .get_inputs_for_target(&target_user.target) + .is_some_and(|inputs| inputs.iter().any(|candidate| candidate.name == input.name)) + }) + .map(|target_user| { + target_user + .credentials + .iter() + .map(|user| { + if user.max_connections == 0 { + u32::MAX + } else { + user.max_connections.saturating_add(u32::from(user.soft_connections)) + } + }) + .fold(0u32, u32::saturating_add) + }) + .max() + .unwrap_or(1) + .max(1) +} + +async fn hls_active_tuliprox_target_user_connections_for_input(app_state: &Arc, input: &ConfigInput) -> u32 { + u32::try_from( + app_state + .active_users + .active_streams() + .await + .iter() + .filter(|stream| stream.channel.input_name == input.name) + .count(), + ) + .unwrap_or(u32::MAX) +} + +fn hls_soft_overlap_delay_ms( + target_duration_ms: u64, + tuliprox_target_user_connection_capacity: u32, + origin_input_account_connection_capacity: u32, +) -> u64 { + let target_duration_ms = target_duration_ms.max(1); + let users = u64::from(tuliprox_target_user_connection_capacity.max(1)); + let origin = u64::from(origin_input_account_connection_capacity.max(1)); + if users >= origin.saturating_mul(2) { + return target_duration_ms; + } + if users <= origin { + return target_duration_ms.saturating_mul(2); + } + let numerator = origin.saturating_mul(3).saturating_sub(users); + target_duration_ms + .saturating_mul(numerator) + .saturating_add(origin.saturating_sub(1)) + / origin +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn prepare_hls_origin_runtime( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + promote_elapsed_hls_account_overlaps(app_state, now_ms).await; + detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; + reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; + let existing_binding = session.read().await.origin_account_binding.clone(); + let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached); + let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref()); + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_started(session, work_kind).await; + } + if let Some(binding) = existing_binding { + if binding.is_active() { + match hls_origin_account_status(app_state, &binding) { + stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => { + return rebind_hls_origin_account( + app_state, + session, + input, + raw_request_url, + session_entry_url, + &binding, + stale_status, + fingerprint, + connection_kind, + priority, + now_ms, + ) + .await; + } + HlsOriginAccountStatus::Known => { + return Ok(prepared_hls_origin_runtime_for_known_binding( + app_state, + input, + raw_request_url, + session_entry_url, + &binding, + )); + } + } + } + } + + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + false, + work_kind, + work_class, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)), + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + + if work_class.allows_speculative_overlap() { + if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime( + app_state, + session, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + + if let Ok(prepared) = prepare_hls_speculative_origin_runtime( + app_state, + session, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + } else { + debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value()); + } + + if work_class.allows_grace() { + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + true, + work_kind, + work_class, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { + return Err(HlsOriginRuntimeAcquireError::Fatal(status)) + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + } else { + debug!( + "HLS origin account grace skipped: work_class={} reason=background-origin-work", + work_class.as_log_value() + ); + } + + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_failed(session, "no-account-available").await; + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason }) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_hls_origin_runtime_with_new_account( + app_state: &Arc, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + allow_grace: bool, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + let session_owner = build_hls_origin_session_owner(proxy_session_id); + let Some(provider_handle) = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &fingerprint.addr, + allow_grace, + priority, + connection_kind, + Some(&session_owner), + ) + .await + else { + debug!( + "HLS origin account acquire unavailable: work={} work_class={} grace={}", + work_kind.as_log_value(), + work_class.as_log_value(), + if allow_grace { "attempted" } else { "disabled" } + ); + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, + }); + }; + + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(fetch_url) = build_hls_origin_fetch_url(input, raw_request_url, session_entry_url, Some(&provider_config)) + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(binding) = origin_account_binding_from_allocation( + Arc::clone(&input.name), + proxy_session_id, + &provider_handle.allocation, + now_ms, + ) else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let grace_state = if matches!(provider_handle.allocation, ProviderAllocation::GracePeriod(_)) { + "granted" + } else if allow_grace { + "not-needed" + } else { + "disabled" + }; + debug!( + "HLS origin account binding created: account={} owner={} work={} work_class={} grace={}", + sanitize_sensitive_info(binding.account_name.as_ref()), + sanitize_sensitive_info(&binding.session_owner), + work_kind.as_log_value(), + work_class.as_log_value(), + grace_state + ); + + Ok(PreparedHlsOriginRuntime { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + fetch_url, + origin_account_binding_to_store: Some(binding), + preacquired_origin_account_handle: Some(provider_handle), + }) +} + +fn prepared_hls_origin_runtime_for_known_binding( + app_state: &Arc, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + binding: &HlsOriginAccountBinding, +) -> PreparedHlsOriginRuntime { + let fetch_url = app_state + .active_provider + .find_provider_config(&binding.account_name) + .as_ref() + .and_then(|provider_config| { + build_hls_origin_fetch_url(input, raw_request_url, session_entry_url, Some(provider_config)) + }) + .unwrap_or_else(|| session_entry_url.to_string()); + + PreparedHlsOriginRuntime { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + fetch_url, + origin_account_binding_to_store: None, + preacquired_origin_account_handle: None, + } +} + +async fn log_hls_origin_binding_reacquire_started(session: &HlsSessionHandle, work_kind: HlsOriginWorkKind) { + let session_guard = session.read().await; + let mode = match session_guard.mode { + HlsSessionMode::NormalCacheTimeline => "normal", + HlsSessionMode::TransientPassthrough { .. } => "transient", + }; + debug!( + "HLS origin binding reacquire started: session={} mode={} work={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + mode, + work_kind.as_log_value() + ); +} + +async fn log_hls_origin_binding_reacquired(session: &HlsSessionHandle, binding: &HlsOriginAccountBinding) { + let session_guard = session.read().await; + debug!( + "HLS origin binding reacquired: session={} account={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(binding.account_name.as_ref()) + ); +} + +async fn log_hls_origin_binding_reacquire_failed(session: &HlsSessionHandle, reason: &str) { + let session_guard = session.read().await; + debug!( + "HLS origin binding reacquire failed: session={} reason={} retry_after_ms={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + reason, + cold_start_retry_after_seconds().saturating_mul(1_000) + ); +} + +async fn detach_unprotected_hls_origin_account_bindings(app_state: &Arc, now_ms: u64) { + let sessions = app_state.hls_proxy.sessions().list_sessions().await; + for session in sessions { + let binding = { + let mut session_guard = session.write().await; + let Some(binding) = session_guard.origin_account_binding.clone() else { + continue; + }; + if !matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active) { + continue; + } + let timing = session_guard.account_overlap_timing(); + let protection = session_guard.account_binding_protection(now_ms); + debug!( + "HLS account protection classified: session={} state={} target_duration_ms={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + protection.as_log_state(), + timing.target_duration_ms + ); + if !matches!(protection, HlsAccountBindingProtection::Expired) + || session_guard.activity.active_origin_work_count > 0 + { + continue; + } + if !matches!(hls_origin_account_status(app_state, &binding), HlsOriginAccountStatus::Known) { + continue; + } + if let Some(binding) = session_guard.origin_account_binding.as_mut() { + binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, now_ms); + } + session_guard.invalidate_queued_origin_work(); + debug!( + "HLS origin binding detached: session={} account={} reason={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(binding.account_name.as_ref()), + HlsOriginAccountDetachedReason::SoftWindowElapsed.as_log_reason() + ); + binding + }; + app_state.active_provider.clear_provider_reservation(&binding.session_owner).await; + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn prepare_hls_origin_policy_preempt_runtime( + app_state: &Arc, + new_session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + now_ms: u64, +) -> Result { + let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms); + let Some(candidate) = + find_hls_origin_policy_preempt_candidate(app_state, input, proxy_session_id, request_policy, now_ms).await + else { + debug!("HLS origin policy preemption denied: reason=no-lower-origin-policy-candidate"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + app_state.active_provider.clear_provider_reservation(&candidate.session_owner).await; + let session_owner = build_hls_origin_session_owner(proxy_session_id); + let Some(provider_handle) = app_state + .active_provider + .acquire_exact_connection_with_grace_for_session( + &candidate.account_name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(&session_owner), + ) + .await + else { + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + debug!("HLS origin policy preemption denied: reason=exact-acquire-failed"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + debug!("HLS origin policy preemption denied: reason=missing-provider-config"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let Some(fetch_url) = build_hls_origin_fetch_url(input, raw_request_url, session_entry_url, Some(&provider_config)) + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + debug!("HLS origin policy preemption denied: reason=invalid-origin-url"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let Some(binding) = origin_account_binding_from_allocation( + Arc::clone(&input.name), + proxy_session_id, + &provider_handle.allocation, + now_ms, + ) else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + debug!("HLS origin policy preemption denied: reason=invalid-allocation"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let mut detached_victim = false; + { + let mut victim = candidate.session.write().await; + if let Some(victim_binding) = victim.origin_account_binding.as_mut() { + if victim_binding.account_name == candidate.account_name + && victim_binding.session_owner == candidate.session_owner + && matches!(victim_binding.binding_mode, HlsOriginAccountBindingMode::Active) + { + victim_binding.detach(HlsOriginAccountDetachedReason::PreemptedByHigherPriority, now_ms); + detached_victim = true; + } + } + if detached_victim { + victim.invalidate_queued_origin_work(); + } + } + if !detached_victim { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate).await; + debug!("HLS origin policy preemption denied: reason=stale-candidate"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + + { + let mut session_guard = new_session.write().await; + session_guard.replace_origin_account_binding(Some(binding.clone())); + } + debug!( + "HLS origin policy preempted account {}: session {} yielded to session {}; victim_kind={:?} victim_priority={} request_kind={:?} request_priority={}", + sanitize_sensitive_info(candidate.account_name.as_ref()), + safe_proxy_session_id(&candidate.proxy_session_id), + safe_proxy_session_id(proxy_session_id), + candidate.victim_policy.connection_kind, + candidate.victim_policy.priority, + request_policy.connection_kind, + request_policy.priority + ); + debug!( + "HLS origin binding detached: session={} account={} reason={}", + safe_proxy_session_id(&candidate.proxy_session_id), + sanitize_sensitive_info(candidate.account_name.as_ref()), + HlsOriginAccountDetachedReason::PreemptedByHigherPriority.as_log_reason() + ); + + Ok(PreparedHlsOriginRuntime { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + fetch_url, + origin_account_binding_to_store: Some(binding), + preacquired_origin_account_handle: Some(provider_handle), + }) +} + +async fn restore_hls_origin_policy_preempt_candidate_reservation( + app_state: &Arc, + candidate: &HlsOriginPolicyPreemptCandidate, +) { + app_state + .active_provider + .refresh_provider_reservation( + &candidate.account_name, + &candidate.session_owner, + candidate.reservation_ttl_secs, + ) + .await; +} + +async fn find_hls_origin_policy_preempt_candidate( + app_state: &Arc, + input: &ConfigInput, + new_proxy_session_id: &ProxySessionId, + request_policy: HlsEffectiveOriginAcquirePolicy, + _now_ms: u64, +) -> Option { + let sessions = app_state.hls_proxy.sessions().list_sessions().await; + let mut best_candidate = None; + for session in sessions { + let session_guard = session.read().await; + if session_guard.proxy_session_id == *new_proxy_session_id { + continue; + } + let Some(binding) = session_guard.origin_account_binding.as_ref() else { + continue; + }; + if binding.input_name != input.name || !matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active) { + continue; + } + if session_guard.activity.active_origin_work_count > 0 { + continue; + } + if !matches!(hls_origin_account_status(app_state, binding), HlsOriginAccountStatus::Known) { + continue; + } + let victim_policy = session_guard.effective_origin_acquire_policy_or_default(); + if !request_policy.is_better_than(victim_policy) { + continue; + } + let candidate = HlsOriginPolicyPreemptCandidate { + session: Arc::clone(&session), + proxy_session_id: session_guard.proxy_session_id.clone(), + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + reservation_ttl_secs: session_guard.account_overlap_timing().reservation_ttl_secs(), + victim_policy, + last_media_at_ms: session_guard.activity.last_authorized_media_at_ms.unwrap_or_default(), + }; + if hls_origin_policy_preempt_candidate_is_better(best_candidate.as_ref(), &candidate) { + best_candidate = Some(candidate); + } + } + best_candidate +} + +fn hls_origin_policy_preempt_candidate_is_better( + current: Option<&HlsOriginPolicyPreemptCandidate>, + candidate: &HlsOriginPolicyPreemptCandidate, +) -> bool { + let Some(current) = current else { + return true; + }; + match (candidate.victim_policy.connection_kind, current.victim_policy.connection_kind) { + (crate::api::model::ConnectionKind::Soft, crate::api::model::ConnectionKind::Normal) => return true, + (crate::api::model::ConnectionKind::Normal, crate::api::model::ConnectionKind::Soft) => return false, + _ => {} + } + candidate.victim_policy.priority > current.victim_policy.priority + || (candidate.victim_policy.priority == current.victim_policy.priority + && candidate.last_media_at_ms < current.last_media_at_ms) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_hls_speculative_origin_runtime( + app_state: &Arc, + new_session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + now_ms: u64, +) -> Result { + let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else { + debug!("HLS account overlap denied: reason=no-soft-active-candidate"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + app_state.active_provider.clear_provider_reservation(&candidate.session_owner).await; + let session_owner = build_hls_origin_session_owner(proxy_session_id); + let Some(provider_handle) = app_state + .active_provider + .acquire_exact_connection_with_grace_for_session( + &candidate.account_name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(&session_owner), + ) + .await + else { + debug!("HLS account overlap denied: reason=speculative-acquire-failed"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + debug!("HLS account overlap denied: reason=missing-provider-config"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let Some(fetch_url) = build_hls_origin_fetch_url(input, raw_request_url, session_entry_url, Some(&provider_config)) + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + debug!("HLS account overlap denied: reason=invalid-origin-url"); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let binding = HlsOriginAccountBinding::speculative_from( + Arc::clone(&input.name), + Arc::clone(&candidate.account_name), + proxy_session_id, + candidate.proxy_session_id.clone(), + candidate.reclaim_until_ms, + now_ms, + ); + { + let mut session_guard = new_session.write().await; + session_guard.replace_origin_account_binding(Some(binding.clone())); + } + debug!( + "HLS account overlap granted for account {}: session {} yields to session {}; reclaim until {} ms, eligible after {} ms, delay {} ms (Tuliprox target user connections {}, origin input account connections {})", + sanitize_sensitive_info(candidate.account_name.as_ref()), + safe_proxy_session_id(&candidate.proxy_session_id), + safe_proxy_session_id(proxy_session_id), + candidate.reclaim_until_ms, + candidate.soft_overlap_eligible_at_ms, + candidate.soft_overlap_delay_ms, + candidate.tuliprox_target_user_connection_capacity, + candidate.origin_input_account_connection_capacity + ); + Ok(PreparedHlsOriginRuntime { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + fetch_url, + origin_account_binding_to_store: Some(binding), + preacquired_origin_account_handle: Some(provider_handle), + }) +} + +async fn find_hls_account_overlap_candidate( + app_state: &Arc, + input: &ConfigInput, + new_proxy_session_id: &ProxySessionId, + now_ms: u64, +) -> Option { + let sessions = app_state.hls_proxy.sessions().list_sessions().await; + let tuliprox_target_user_connection_capacity = + hls_tuliprox_target_user_connection_capacity(app_state, input).await; + let origin_input_account_connection_capacity = hls_origin_input_account_connection_capacity(app_state, input).await; + let mut speculative_accounts = Vec::new(); + for session in &sessions { + let session = session.read().await; + let Some(binding) = session.origin_account_binding.as_ref() else { + continue; + }; + if binding.input_name != input.name { + continue; + } + if matches!( + binding.binding_mode, + HlsOriginAccountBindingMode::Speculative { reclaim_until_ms, .. } if now_ms <= reclaim_until_ms + ) { + speculative_accounts.push(Arc::clone(&binding.account_name)); + } + } + + let mut candidates = Vec::new(); + for session in sessions { + let session_guard = session.read().await; + if session_guard.proxy_session_id == *new_proxy_session_id { + continue; + } + let Some(binding) = session_guard.origin_account_binding.as_ref() else { + continue; + }; + if binding.input_name != input.name + || speculative_accounts.iter().any(|account| account == &binding.account_name) + { + continue; + } + if !matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active) { + continue; + } + if session_guard.activity.active_origin_work_count > 0 { + continue; + } + let timing = session_guard.account_overlap_timing(); + let protection = session_guard.account_binding_protection(now_ms); + debug!( + "HLS account protection classified: session={} state={} target_duration_ms={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + protection.as_log_state(), + timing.target_duration_ms + ); + let HlsAccountBindingProtection::SoftActive { reclaim_until_ms } = protection else { + continue; + }; + let last_media_at_ms = session_guard.activity.last_authorized_media_at_ms.unwrap_or_default(); + let capacity = hls_soft_overlap_capacity_for_target_duration( + tuliprox_target_user_connection_capacity, + origin_input_account_connection_capacity, + timing.target_duration_ms, + ); + let eligible_at_ms = last_media_at_ms.saturating_add(capacity.delay_ms); + if now_ms < eligible_at_ms { + debug!( + "HLS account overlap waits for session {} account {}; eligible at {} ms, now {} ms, delay {} ms (Tuliprox target user connections {}, origin input account connections {})", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(binding.account_name.as_ref()), + eligible_at_ms, + now_ms, + capacity.delay_ms, + capacity.tuliprox_target_user_connection_capacity, + capacity.origin_input_account_connection_capacity + ); + continue; + } + candidates.push(HlsAccountOverlapCandidate { + proxy_session_id: session_guard.proxy_session_id.clone(), + input_name: Arc::clone(&binding.input_name), + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + reclaim_until_ms, + last_media_at_ms, + soft_overlap_eligible_at_ms: eligible_at_ms, + soft_overlap_delay_ms: capacity.delay_ms, + tuliprox_target_user_connection_capacity: capacity.tuliprox_target_user_connection_capacity, + origin_input_account_connection_capacity: capacity.origin_input_account_connection_capacity, + }); + } + let mut eligible = filter_hls_account_overlap_cooldowns(app_state, candidates, now_ms).await; + eligible.sort_by_key(|candidate| (candidate.last_media_at_ms, candidate.soft_overlap_eligible_at_ms)); + eligible.into_iter().next() +} + +async fn filter_hls_account_overlap_cooldowns( + app_state: &Arc, + candidates: Vec, + now_ms: u64, +) -> Vec { + let mut eligible = Vec::new(); + for candidate in candidates { + if app_state + .hls_proxy + .is_account_overlap_cooling_down(&candidate.input_name, &candidate.account_name, now_ms) + .await + { + debug!( + "HLS account overlap skips session {} account {}; cooldown is still active", + safe_proxy_session_id(&candidate.proxy_session_id), + sanitize_sensitive_info(candidate.account_name.as_ref()) + ); + continue; + } + eligible.push(candidate); + } + eligible +} + +async fn reclaim_hls_account_overlap_if_needed( + app_state: &Arc, + winner_session: &HlsSessionHandle, + now_ms: u64, +) { + let winner_proxy_session_id = winner_session.read().await.proxy_session_id.clone(); + let sessions = app_state.hls_proxy.sessions().list_sessions().await; + for session in sessions { + let (loser_proxy_session_id, loser_binding) = { + let session_guard = session.read().await; + let Some(binding) = session_guard.origin_account_binding.clone() else { + continue; + }; + let HlsOriginAccountBindingMode::Speculative { displaced_proxy_session_id, reclaim_until_ms } = + &binding.binding_mode + else { + continue; + }; + if displaced_proxy_session_id != &winner_proxy_session_id || now_ms > *reclaim_until_ms { + continue; + } + (session_guard.proxy_session_id.clone(), binding) + }; + app_state.active_provider.clear_provider_reservation(&loser_binding.session_owner).await; + { + let mut loser = session.write().await; + if let Some(binding) = loser.origin_account_binding.as_mut() { + binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, now_ms); + } + loser.invalidate_queued_origin_work(); + } + { + let mut winner = winner_session.write().await; + if let Some(binding) = winner.origin_account_binding.as_mut() { + binding.promote_to_active(); + } + } + let hard_active_window_ms = winner_session.read().await.account_overlap_timing().hard_active_window_ms; + app_state + .hls_proxy + .mark_account_overlap_reclaimed_cooldown( + Arc::clone(&loser_binding.input_name), + Arc::clone(&loser_binding.account_name), + now_ms, + hard_active_window_ms, + ) + .await; + debug!( + "HLS account overlap reclaimed: account={} winner={} loser={}", + sanitize_sensitive_info(loser_binding.account_name.as_ref()), + safe_proxy_session_id(&winner_proxy_session_id), + safe_proxy_session_id(&loser_proxy_session_id) + ); + debug!( + "HLS origin binding detached: session={} account={} reason={}", + safe_proxy_session_id(&loser_proxy_session_id), + sanitize_sensitive_info(loser_binding.account_name.as_ref()), + HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner.as_log_reason() + ); + } +} + +async fn promote_elapsed_hls_account_overlaps(app_state: &Arc, now_ms: u64) { + let sessions = app_state.hls_proxy.sessions().list_sessions().await; + for session in sessions { + let (input_name, account_name, promoted_session_id, displaced_session_id, hard_active_window_ms) = { + let mut session_guard = session.write().await; + let hard_active_window_ms = session_guard.account_overlap_timing().hard_active_window_ms; + let Some(binding) = session_guard.origin_account_binding.as_mut() else { + continue; + }; + let HlsOriginAccountBindingMode::Speculative { displaced_proxy_session_id, reclaim_until_ms } = + &binding.binding_mode + else { + continue; + }; + if now_ms <= *reclaim_until_ms { + continue; + } + let displaced_session_id = displaced_proxy_session_id.clone(); + let input_name = Arc::clone(&binding.input_name); + let account_name = Arc::clone(&binding.account_name); + binding.promote_to_active(); + ( + input_name, + account_name, + session_guard.proxy_session_id.clone(), + displaced_session_id, + hard_active_window_ms, + ) + }; + app_state + .hls_proxy + .mark_account_overlap_promoted_cooldown( + Arc::clone(&input_name), + Arc::clone(&account_name), + now_ms, + hard_active_window_ms, + ) + .await; + if let Some(displaced) = app_state.hls_proxy.sessions().get_by_proxy_session_id(&displaced_session_id).await { + let mut detached = false; + let mut displaced = displaced.write().await; + if displaced.origin_account_binding.as_ref().is_some_and(|binding| binding.account_name == account_name) { + if let Some(binding) = displaced.origin_account_binding.as_mut() { + binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, now_ms); + detached = true; + } + displaced.invalidate_queued_origin_work(); + } + if detached { + debug!( + "HLS origin binding detached: session={} account={} reason={}", + safe_proxy_session_id(&displaced_session_id), + sanitize_sensitive_info(account_name.as_ref()), + HlsOriginAccountDetachedReason::SoftWindowElapsed.as_log_reason() + ); + } + } + debug!( + "HLS account overlap promoted: account={} session={}", + sanitize_sensitive_info(account_name.as_ref()), + safe_proxy_session_id(&promoted_session_id) + ); + } +} + +#[allow(clippy::too_many_arguments)] +async fn rebind_hls_origin_account( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + stale_binding: &HlsOriginAccountBinding, + stale_status: HlsOriginAccountStatus, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + now_ms: u64, +) -> Result { + { + let mut session_guard = session.write().await; + if !session_guard.origin_account_rebind.is_allowed_now(now_ms) { + debug!( + "HLS origin account rebind skipped by backoff: session={} old_account={} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms); + } + + let safe_session_id = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + debug!( + "HLS origin account rebind started: session={} old_account={} reason={stale_status:?}", + safe_session_id, + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + app_state.active_provider.clear_provider_reservation(&stale_binding.session_owner).await; + { + let mut session_guard = session.write().await; + if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { + binding.account_name == stale_binding.account_name && binding.session_owner == stale_binding.session_owner + }) { + binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); + debug!( + "HLS origin binding detached: session={} account={} reason={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason() + ); + } + session_guard.invalidate_queued_origin_work(); + } + + let Some(provider_handle) = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(&stale_binding.session_owner), + ) + .await + else { + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(Some(stale_binding)), + }); + }; + + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(fetch_url) = build_hls_origin_fetch_url(input, raw_request_url, session_entry_url, Some(&provider_config)) + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(new_account_name) = provider_handle.allocation.get_provider_name() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let new_binding = HlsOriginAccountBinding::rebound( + Arc::clone(&input.name), + new_account_name, + stale_binding.session_owner.clone(), + stale_binding.generation.saturating_add(1), + now_ms, + ); + { + let mut session_guard = session.write().await; + session_guard.replace_origin_account_binding(Some(new_binding.clone())); + session_guard.origin_account_rebind.mark_success(); + } + debug!( + "HLS origin account rebound: old_account={} new_account={}", + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + sanitize_sensitive_info(new_binding.account_name.as_ref()) + ); + + Ok(PreparedHlsOriginRuntime { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + fetch_url, + origin_account_binding_to_store: None, + preacquired_origin_account_handle: Some(provider_handle), + }) +} + +async fn mark_hls_origin_rebind_failed( + session: &HlsSessionHandle, + stale_binding: &HlsOriginAccountBinding, + now_ms: u64, + reason: &str, +) { + let mut session_guard = session.write().await; + session_guard.origin_account_rebind.mark_failed(now_ms); + debug!( + "HLS origin account rebind failed: session={} old_account={} reason={reason} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_hls_cache_user_session( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + session_token: &str, + virtual_id: u32, + request_url: &str, + input: &ConfigInput, + connection_permission: UserConnectionPermission, + connection_kind: Option, +) -> String { + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id, + provider: input.name.as_ref(), + stream_url: request_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind, + socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), + }) + .await +} + +fn hls_entry_origin_connection_kind( + connection_permission: UserConnectionPermission, + connection_kind: Option, +) -> Option { + match connection_permission { + UserConnectionPermission::Allowed | UserConnectionPermission::GracePeriod => connection_kind, + UserConnectionPermission::Exhausted => None, + } +} + +#[allow(clippy::too_many_arguments)] +async fn try_hls_cache_entry_redirect( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + origin_source: HlsOriginSource, + virtual_id: u32, + _existing_user_session: Option<&UserSession>, + request_url: &str, + input: &ConfigInput, + connection_permission: UserConnectionPermission, + connection_kind: Option, + server_path: Option<&str>, +) -> Option { + if !hls_cache_configured(app_state) { + return None; + } + + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let family_key = HlsPlaybackFamilyKey::new(user.username.clone(), fingerprint.key.clone()); + let now_ms = current_time_millis(); + let origin_connection_kind = hls_entry_origin_connection_kind(connection_permission, connection_kind); + let access_lease_id = new_hls_access_lease_id(); + let session_token = create_hls_cache_user_session_token(fingerprint, &user.username, virtual_id); + let session_token = prepare_hls_cache_user_session( + app_state, + fingerprint, + user, + &session_token, + virtual_id, + request_url, + input, + connection_permission, + origin_connection_kind, + ) + .await; + let mut lease = HlsAccessLease::pending( + access_lease_id.clone(), + family_key, + proxy_session_id.clone(), + user.username.clone(), + session_token.clone(), + origin_source.input_id, + origin_source.stream_ref.clone(), + virtual_id, + now_ms, + hls_pending_bootstrap_window_ms(), + ); + if let Some(connection_kind) = origin_connection_kind { + lease = lease.with_origin_acquire_policy(connection_kind, connection_priority_for_kind(user, connection_kind)); + } else { + lease.state = HlsAccessLeaseState::Denied; + } + app_state.hls_proxy.prepare_access_lease(lease).await; + debug!( + "HLS access lease prepared: lease={} proxy_session={} session={} action=created reason=new-playback", + safe_hls_access_lease_id(&access_lease_id), + safe_proxy_session_id(&proxy_session_id), + safe_user_session_token(&session_token) + ); + Some(hls_canonical_manifest_redirect(&proxy_session_id, &access_lease_id, server_path)) +} + +fn hls_canonical_manifest_redirect( + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + server_path: Option<&str>, +) -> axum::response::Response { + let path_prefix = normalize_hls_proxy_public_path_prefix(server_path).unwrap_or_default(); + let location = format!("{path_prefix}{}", hls_canonical_manifest_path(proxy_session_id, access_lease_id)); + try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::TEMPORARY_REDIRECT) + .header(header::LOCATION, location) + .header(header::CACHE_CONTROL, "no-store") + .body(Body::empty())) +} + +fn hls_canonical_manifest_path(proxy_session_id: &ProxySessionId, access_lease_id: &HlsAccessLeaseId) -> String { + format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, access_lease_id.0) +} + +fn hls_canonical_retry_after_response() -> axum::response::Response { + try_unwrap_body!(axum::response::Response::builder() + .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) + .header(axum::http::header::RETRY_AFTER, cold_start_retry_after_seconds().to_string()) + .body(axum::body::Body::empty())) +} + +fn hls_canonical_status_response(status: StatusCode) -> axum::response::Response { + if status == StatusCode::SERVICE_UNAVAILABLE { + hls_canonical_retry_after_response() + } else { + status.into_response() + } +} + +struct HlsEntryOriginAccountReservation { + request_url: String, + session_token: String, + provider_handle: Option, + selected_provider_config: Option>, +} + +#[allow(clippy::too_many_arguments)] +async fn try_reserve_hls_entry_origin_account_for_redirect( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + input: &ConfigInput, + virtual_id: u32, + request_url: &str, + user_session_token: &str, + session_owner: &str, + reservation_ttl_secs: u64, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + create_user_session: bool, +) -> Option { + let provider_handle = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &fingerprint.addr, + false, + connection_priority_for_kind(user, connection_kind), + connection_kind, + Some(session_owner), + ) + .await?; + + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return None; + }; + let Some(stream_url) = get_stream_alternative_url(request_url, input, &provider_config) else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return None; + }; + + let session_token = if create_user_session { + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token: user_session_token, + virtual_id, + provider: &provider_config.name, + stream_url: &stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), + }) + .await + } else { + user_session_token.to_string() + }; + + app_state + .active_provider + .refresh_provider_reservation(&provider_config.name, session_owner, reservation_ttl_secs) + .await; + + Some(HlsEntryOriginAccountReservation { + request_url: stream_url, + session_token, + provider_handle: Some(provider_handle), + selected_provider_config: Some(provider_config), + }) +} + +#[allow(clippy::too_many_arguments)] +async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + virtual_id: u32, +) -> bool { + let session_token = + create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, PlaylistItemType::LiveHls, None); + let (connection_admission, _, _) = resolve_playback_request_admission( + app_state, + user, + fingerprint, + PlaylistItemType::LiveHls, + None, + &session_token, + false, + EvictionReentryGuard::SocketPlayback { virtual_id }, + false, + false, + ) + .await; + if connection_admission.permission == UserConnectionPermission::Exhausted { + return false; + } + + let Some(channel) = get_stream_channel(app_state, target, virtual_id).await else { + return false; + }; + let Ok(origin_playlist_url) = + resolve_hls_origin_playlist_url(app_state, target, input, virtual_id, channel.url.as_ref()).await + else { + return false; + }; + let Some(hls_cache_origin) = build_hls_origin_resolution(input, &origin_playlist_url) else { + return false; + }; + let Some(connection_kind) = connection_admission.kind else { + return false; + }; + let (shared_hls_session_owner, reservation_ttl_secs) = if hls_cache_enabled_for_target(app_state, target) { + let origin_source = build_hls_origin_source(input, hls_stream_ref_from_virtual_id(virtual_id)); + let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + let reservation_ttl_secs = match app_state.hls_proxy.sessions().get_by_key(&origin_source.session_key()).await { + Some(session) => hls_origin_account_reservation_ttl_secs_for_session(&session).await, + None => hls_origin_account_reservation_ttl_secs_fallback(), + }; + (Some(build_hls_origin_session_owner(&proxy_session_id)), reservation_ttl_secs) + } else { + (None, get_hls_session_ttl_secs(app_state)) + }; + let session_owner = shared_hls_session_owner.as_deref().unwrap_or(session_token.as_str()); + + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + input, + virtual_id, + hls_cache_origin.session_entry_url.as_str(), + &session_token, + session_owner, + reservation_ttl_secs, + connection_admission.permission, + connection_kind, + false, + ) + .await + else { + return false; + }; + + app_state.connection_manager.release_provider_handle(reservation.provider_handle).await; + true +} + +async fn mark_hls_provisioning_handoff_discontinuity( + app_state: &Arc, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: Option<&HlsAccessLeaseId>, + now_ms: u64, +) -> bool { + if !hls_cache_configured(app_state) { + return false; + } + let origin_source = build_hls_origin_source(input, hls_stream_ref_from_virtual_id(virtual_id)); + let Some(session) = app_state.hls_proxy.sessions().get_by_key(&origin_source.session_key()).await else { + return false; + }; + mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state, + &session, + input, + virtual_id, + access_lease_id, + now_ms, + ) + .await +} + +async fn mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: Option<&HlsAccessLeaseId>, + now_ms: u64, +) -> bool { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + if !app_state.hls_provisioning.mark_handoff_once( + &input.name, + virtual_id, + Some(&proxy_session_id), + access_lease_id, + now_ms, + ) { + debug!( + "HLS provisioning handoff discontinuity already marked: proxy_session_id={}", + safe_proxy_session_id(&proxy_session_id) + ); + return false; + } + mark_hls_provisioning_handoff_discontinuity_for_session(session, now_ms).await; + ensure_shared_hls_provisioning_handoff_gap(app_state, session, now_ms).await; + true +} + +async fn mark_hls_provisioning_handoff_discontinuity_for_session(session: &HlsSessionHandle, now_ms: u64) { + let discontinuity_sequence = hls_provisioning_discontinuity_sequence(now_ms); + let proxy_session_id = { + let mut session = session.write().await; + session.mark_pending_handoff_discontinuity(discontinuity_sequence); + session.proxy_session_id.clone() + }; + debug!( + "HLS provisioning handoff discontinuity marked: proxy_session_id={} discontinuity_sequence={}", + safe_proxy_session_id(&proxy_session_id), + discontinuity_sequence + ); +} + +fn clear_hls_provisioning_handoff_consumer( + app_state: &Arc, + input: &ConfigInput, + virtual_id: u32, + now_ms: u64, +) { + if !app_state.hls_provisioning.take_ready_slot_for_consumer(&input.name, virtual_id, now_ms) { + app_state.hls_provisioning.clear_consumer(&input.name, virtual_id); + } +} + +async fn maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + now_ms: u64, +) -> Option { + if !app_state.hls_provisioning.has_consumer(&input.name, virtual_id, now_ms) { + return None; + } + let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(session).await; + mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state, + session, + input, + virtual_id, + Some(access_lease_id), + now_ms, + ) + .await + .then_some(previous_manifest_rendered_at_ms) +} + +async fn latest_shared_hls_manifest_rendered_at_ms(session: &HlsSessionHandle) -> u64 { + let session = session.read().await; + session + .last_rendered_manifest + .as_ref() + .map_or(0, |rendered| rendered.rendered_at_ms) + .max(session.transient.last_manifest_rendered_at_ms.unwrap_or(0)) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api) async fn hls_panel_provisioning_poll_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + virtual_id: u32, + original_hls_entry_path: &str, + server_path: Option<&str>, +) -> axum::response::Response { + hls_panel_provisioning_poll_response( + app_state, + fingerprint, + user, + target, + input, + virtual_id, + original_hls_entry_path, + server_path, + HlsProvisioningPollResponseKind::Legacy, + ) + .await +} + +enum HlsProvisioningPollResponseKind { + Legacy, +} + +impl HlsProvisioningPollResponseKind { + fn access_lease_id(&self) -> Option<&HlsAccessLeaseId> { + match self { + Self::Legacy => None, + } + } +} + +#[allow(clippy::too_many_arguments)] +async fn hls_panel_provisioning_poll_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + virtual_id: u32, + ready_redirect_path: &str, + server_path: Option<&str>, + response_kind: HlsProvisioningPollResponseKind, +) -> axum::response::Response { + let now_ms = current_time_millis(); + app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + + let existing_status = app_state.hls_provisioning.consumer_status(&input.name, virtual_id, now_ms); + + if try_reserve_hls_virtual_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + target, + input, + virtual_id, + ) + .await + { + mark_hls_provisioning_handoff_discontinuity( + app_state, + input, + virtual_id, + response_kind.access_lease_id(), + now_ms, + ) + .await; + clear_hls_provisioning_handoff_consumer(app_state, input, virtual_id, current_time_millis()); + return hls_virtual_entry_redirect_response(ready_redirect_path, server_path); + } + + let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); + if provisioning_enabled { + start_hls_panel_provisioning_once(app_state, input); + } + + let status = existing_status.unwrap_or(if provisioning_enabled { + HlsProvisioningStatus::InProgress + } else { + HlsProvisioningStatus::ProviderExhausted + }); + + match status { + HlsProvisioningStatus::Ready | HlsProvisioningStatus::InProgress => hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::Provisioning, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ), + HlsProvisioningStatus::ProviderExhausted => hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ), + } +} + +async fn hls_panel_provisioning_or_status_response( + app_state: &Arc, + user: &ProxyUserCredentials, + input: &ConfigInput, + virtual_id: u32, + _original_hls_entry_path: &str, + server_path: Option<&str>, + fallback_status: StatusCode, +) -> axum::response::Response { + try_hls_panel_provisioning_manifest_response( + app_state, + user, + input, + virtual_id, + HlsPanelProvisioningRedirectPaths { + waiting_manifest_path: None, + }, + server_path, + fallback_status, + ) + .await + .unwrap_or_else(|| fallback_status.into_response()) +} + +#[derive(Debug, Clone)] +struct SharedHlsProvisioningSegmentPlan { + proxy_seq: u64, + physical_index: usize, + cache_key: SegmentCacheKey, + segment_kind: SharedHlsProvisioningLocalSegmentKind, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum SharedHlsProvisioningLocalSegmentKind { + Provisioning, + Gap, +} + +fn shared_hls_provisioning_segment_plans( + session: &HlsSession, + physical_segment_count: usize, +) -> Vec { + let existing_provisioning_segments = session + .segments + .values() + .filter(|entry| is_hls_provisioning_segment(entry)) + .count(); + let append_count = if existing_provisioning_segments == 0 { 3 } else { 1 }; + let start_proxy_seq = session.proxy_next_seq.unwrap_or(0); + (0..append_count) + .filter_map(|offset| { + let proxy_seq = start_proxy_seq.checked_add(u64::try_from(offset).ok()?)?; + if session.segments.contains_key(&proxy_seq) { + return None; + } + Some(SharedHlsProvisioningSegmentPlan { + proxy_seq, + physical_index: (existing_provisioning_segments + offset) % physical_segment_count, + cache_key: SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"), + segment_kind: SharedHlsProvisioningLocalSegmentKind::Provisioning, + }) + }) + .collect() +} + +fn shared_hls_provisioning_segment_entry( + plan: SharedHlsProvisioningSegmentPlan, + content_length: u64, + duration_ms: u64, + now_ms: u64, +) -> crate::api::model::SegmentEntry { + let origin_epoch = match plan.segment_kind { + SharedHlsProvisioningLocalSegmentKind::Provisioning => HLS_PROVISIONING_ORIGIN_EPOCH, + SharedHlsProvisioningLocalSegmentKind::Gap => HLS_PROVISIONING_GAP_ORIGIN_EPOCH, + }; + crate::api::model::SegmentEntry { + origin_key: OriginSegmentKey { origin_epoch, origin_seq: plan.proxy_seq }, + proxy_seq: plan.proxy_seq, + duration_ms, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: plan.cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length, ready_at_ms: now_ms }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + } +} + +async fn commit_shared_hls_provisioning_segments( + app_state: &Arc, + session: &HlsSessionHandle, + plans: &[SharedHlsProvisioningSegmentPlan], + provisioning_segments: &[TransportStreamBuffer], +) -> Option> { + let mut committed = Vec::with_capacity(plans.len()); + for plan in plans { + let video = provisioning_segments.get(plan.physical_index)?; + let duration_ms = video.duration_ms().unwrap_or(HLS_PROVISIONING_SEGMENT_DURATION_MS); + let metadata = match app_state + .hls_proxy + .segment_cache() + .write_bytes_and_commit(&plan.cache_key, video.as_bytes()) + .await + { + Ok(metadata) => metadata, + Err(err) => { + let safe_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + warn!( + "HLS provisioning segment cache commit failed for shared manifest: session={} seq={} error={err}", + safe_session, plan.proxy_seq + ); + return None; + } + }; + committed.push((plan.clone(), metadata.size, duration_ms)); + } + Some(committed) +} + +async fn ensure_shared_hls_provisioning_handoff_gap( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) -> bool { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + let Some(provisioning_segments) = custom_stream_response + .as_ref() + .map(|response| response.panel_api_provisioning_hls_segments.clone()) + .filter(|segments| !segments.is_empty()) + else { + return false; + }; + let plan = { + let session_guard = session.read().await; + if !session_guard.segments.values().any(is_hls_provisioning_segment) + || session_guard.segments.values().any(is_hls_provisioning_gap_segment) + { + return false; + } + let proxy_seq = session_guard.proxy_next_seq.unwrap_or(0); + if session_guard.segments.contains_key(&proxy_seq) { + return false; + } + let existing_provisioning_segments = session_guard + .segments + .values() + .filter(|entry| is_hls_provisioning_segment(entry)) + .count(); + SharedHlsProvisioningSegmentPlan { + proxy_seq, + physical_index: existing_provisioning_segments % provisioning_segments.len(), + cache_key: SegmentCacheKey::new(session_guard.proxy_session_id.clone(), proxy_seq, "ts"), + segment_kind: SharedHlsProvisioningLocalSegmentKind::Gap, + } + }; + let Some(committed) = commit_shared_hls_provisioning_segments( + app_state, + session, + std::slice::from_ref(&plan), + &provisioning_segments, + ) + .await + else { + return false; + }; + let mut session_guard = session.write().await; + let mut inserted = false; + for (plan, content_length, duration_ms) in committed { + if session_guard.segments.contains_key(&plan.proxy_seq) { + continue; + } + if session_guard.publishable_origin_head_proxy_seq.is_none() { + session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); + } + session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); + session_guard + .segments + .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + inserted = true; + } + if inserted { + session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); + session_guard.independent_segments = true; + } + inserted +} + +async fn hls_shared_provisioning_timeline_manifest_response( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, +) -> Option { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + let provisioning_segments = custom_stream_response + .as_ref() + .map(|response| response.panel_api_provisioning_hls_segments.clone()) + .filter(|segments| !segments.is_empty())?; + let now_ms = current_time_millis(); + let plans = { + let session_guard = session.read().await; + shared_hls_provisioning_segment_plans(&session_guard, provisioning_segments.len()) + }; + if plans.is_empty() { + let mut session_guard = session.write().await; + let _ = session_guard.render_and_store_manifest(now_ms).ok()?; + } else { + let committed = + commit_shared_hls_provisioning_segments(app_state, session, &plans, &provisioning_segments).await?; + let mut session_guard = session.write().await; + for (plan, content_length, duration_ms) in committed { + if session_guard.segments.contains_key(&plan.proxy_seq) { + continue; + } + if session_guard.publishable_origin_head_proxy_seq.is_none() { + session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); + } + session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); + session_guard + .segments + .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + } + session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); + session_guard.independent_segments = true; + let _ = session_guard.render_and_store_manifest(now_ms).ok()?; + } + + try_hls_cached_manifest_response( + app_state, + session, + access_lease_id, + access_lease_state, + strip, + server_path, + HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn hls_shared_provisioning_or_provider_exhausted_response( + app_state: &Arc, + session: &HlsSessionHandle, + username: &str, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, +) -> axum::response::Response { + let Some((user, _target)) = app_state.app_config.get_target_for_username(username) else { + return hls_canonical_retry_after_response(); + }; + let now_ms = current_time_millis(); + let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); + if provisioning_enabled { + app_state + .hls_provisioning + .touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + start_hls_panel_provisioning_once(app_state, input); + if let Some(HlsProvisioningStatus::ProviderExhausted) = + app_state.hls_provisioning.consumer_status(&input.name, virtual_id, now_ms) + { + return hls_custom_video_manifest_response( + app_state, + &user, + CustomVideoStreamType::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + ); + } + if let Some(response) = hls_shared_provisioning_timeline_manifest_response( + app_state, + session, + access_lease_id, + access_lease_state, + strip, + server_path, + ) + .await + { + return response; + } + } + + let provider_exhausted_custom_response_available = is_custom_video_stream_enabled(&app_state.app_config) + && app_state + .app_config + .custom_stream_response + .load() + .as_ref() + .and_then(|response| response.provider_connections_exhausted.as_ref()) + .is_some(); + if provider_exhausted_custom_response_available { + return hls_custom_video_manifest_response( + app_state, + &user, + CustomVideoStreamType::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + ); + } + hls_canonical_retry_after_response() +} + +enum HlsProviderExhaustedResolution { + RetryAcquire, + Response(axum::response::Response), +} + +#[allow(clippy::too_many_arguments)] +async fn hls_provider_connections_exhausted_manifest_resolution( + app_state: &Arc, + session: &HlsSessionHandle, + username: &str, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, + allow_grace_hold: bool, +) -> HlsProviderExhaustedResolution { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let grace_options = app_state.get_grace_options(); + if allow_grace_hold && grace_options.hold_stream && grace_options.period_millis > 0 { + debug!( + "HLS provider connections exhausted; holding canonical manifest for grace: session={} lease={} hold_ms={}", + safe_proxy_session_id(&proxy_session_id), + safe_hls_access_lease_id(access_lease_id), + grace_options.period_millis + ); + let capacity_notify = app_state.connection_manager.capacity_notified(); + let deadline = tokio::time::Instant::now() + Duration::from_millis(grace_options.period_millis); + let wake_reason = tokio::select! { + () = capacity_notify.notified() => "capacity-notified", + () = tokio::time::sleep_until(deadline) => "timeout", + }; + debug!( + "HLS provider connections exhausted grace hold completed: session={} lease={} reason={wake_reason}", + safe_proxy_session_id(&proxy_session_id), + safe_hls_access_lease_id(access_lease_id) + ); + return HlsProviderExhaustedResolution::RetryAcquire; + } + + HlsProviderExhaustedResolution::Response( + hls_shared_provisioning_or_provider_exhausted_response( + app_state, + session, + username, + input, + virtual_id, + access_lease_id, + access_lease_state, + strip, + server_path, + ) + .await, + ) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_hls_canonical_manifest_origin_runtime( + app_state: &Arc, + session: &HlsSessionHandle, + context: &HlsAccessContext, + origin: &HlsCacheManifestOrigin<'_>, + path_proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + fingerprint: &Fingerprint, + server_path: Option<&str>, + now_ms: u64, +) -> Result> { + let mut allow_grace_hold = true; + loop { + let origin_policy = hls_effective_origin_acquire_policy(session).await; + match prepare_hls_origin_runtime( + app_state, + session, + origin.input, + origin.raw_request_url, + origin.session_entry_url.as_str(), + path_proxy_session_id, + fingerprint, + origin_policy.connection_kind, + origin_policy.priority, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now_ms, + ) + .await + { + Ok(prepared) => return Ok(prepared), + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, + }) => { + return Err(Box::new(hls_manifest_low_priority_preempted_redirect_response_for_username( + app_state, + &context.username, + ))); + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, + }) => { + let strip = app_state.hls_proxy.strip(); + match hls_provider_connections_exhausted_manifest_resolution( + app_state, + session, + &context.username, + origin.input, + context.virtual_id, + access_lease_id, + access_lease_state, + &strip, + server_path, + allow_grace_hold, + ) + .await + { + HlsProviderExhaustedResolution::RetryAcquire => { + allow_grace_hold = false; + } + HlsProviderExhaustedResolution::Response(response) => return Err(Box::new(response)), + } + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(Box::new(hls_canonical_status_response(status))), + } + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn try_hls_cache_canonical_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + context: &HlsAccessContext, + path_proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + origin: HlsCacheManifestOrigin<'_>, + headers: HeaderMap, + server_path: Option<&str>, + _original_hls_entry_path: &str, +) -> Option { + if !hls_cache_configured(app_state) { + return None; + } + if origin.origin_source.input_id != context.input_id || origin.origin_source.stream_ref != context.stream_ref { + return Some(StatusCode::NOT_FOUND.into_response()); + } + + let session_key = origin.origin_source.session_key(); + let expected_proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + if &expected_proxy_session_id != path_proxy_session_id { + return Some(StatusCode::NOT_FOUND.into_response()); + } + let now_ms = current_time_millis(); + let rewrite_secret = app_state.get_encrypt_secret(); + let (session, session_outcome) = app_state + .hls_proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin.origin_source.clone(), + &rewrite_secret, + now_ms, + ) + .await; + if access_lease_state == HlsAccessLeaseState::Activated { + let timing = hls_access_lease_timing_for_session(app_state, &session).await; + let _ = app_state + .hls_proxy + .touch_manifest_access_lease( + access_lease_id, + path_proxy_session_id, + now_ms, + Some(timing), + None, + hls_access_lease_ttl_ms(app_state), + ) + .await; + } + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + path_proxy_session_id, + now_ms, + ) + .await; + let prepared_origin = match prepare_hls_canonical_manifest_origin_runtime( + app_state, + &session, + context, + &origin, + path_proxy_session_id, + access_lease_id, + access_lease_state, + fingerprint, + server_path, + now_ms, + ) + .await + { + Ok(prepared) => prepared, + Err(response) => return Some(*response), + }; + let url_failover_provider = effective_hls_url_failover_provider_for_fetch_url( + &prepared_origin.fetch_url, + prepared_origin.url_failover_provider.clone(), + origin.session_entry_url.url_failover_provider(), + ); + let origin_entry = + LiveHlsOriginEntry::parse_with_url_failover_provider(&prepared_origin.fetch_url, url_failover_provider)?; + { + let mut session_guard = session.write().await; + if session_guard.is_gc_marked_for_removal() { + return Some(hls_canonical_retry_after_response()); + } + if prepared_origin.origin_account_binding_to_store.is_some() { + session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store); + } + } + mark_hls_authorized_manifest_access(app_state, &session, now_ms).await; + let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else( + || "".to_string(), + |binding| sanitize_sensitive_info(binding.account_name.as_ref()).to_string(), + ); + debug!( + "HLS origin account selected: proxy_session_id={} account={}", + safe_proxy_session_id(path_proxy_session_id), + selected_account + ); + let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(&session).await; + let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(&session).await; + let handoff_previous_rendered_at_ms = maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + app_state, + &session, + origin.input, + context.virtual_id, + access_lease_id, + now_ms, + ) + .await; + let manifest_commit_requirement = + hls_manifest_commit_requirement(&session, session_outcome, handoff_previous_rendered_at_ms, now_ms).await; + let manifest_boundary_rendered_at_ms = + handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); + let wait_timeout = hls_manifest_wait_timeout_for_requirement(&session, manifest_commit_requirement).await; + let cached_manifest_options = hls_cached_manifest_options_for_requirement( + wait_timeout, + manifest_commit_requirement, + manifest_boundary_rendered_at_ms, + ); + + let origin_policy = hls_effective_origin_acquire_policy(&session).await; + let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); + let mut origin_io = HlsOriginIoContext { + app_state: Arc::clone(app_state), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::ManifestInteractive.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + }; + if let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle { + origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + } + + let refresh_request = OriginRefreshRequest { + app_config: Arc::clone(&app_state.app_config), + session: Arc::clone(&session), + origin_entry, + headers, + origin_provider_session_headers, + client: app_state.http_client.load().as_ref().clone(), + no_redirect_client: app_state.http_client_no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + segment_cache: Arc::clone(app_state.hls_proxy.segment_cache()), + hls_proxy: Arc::clone(&app_state.hls_proxy), + segment_repair: Arc::clone(app_state.hls_proxy.segment_repair()), + segment_worker_pool: Arc::clone(app_state.hls_proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(app_state.hls_proxy.map_worker_pool()), + origin_manifest_timeout_ms: app_state.hls_proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: app_state.hls_proxy.manifest_recovery_burst(), + strip: app_state.hls_proxy.strip().clone(), + retry_policy: RetryPolicy::default(), + reverse_proxy_rewrite_secret: rewrite_secret.to_vec(), + transient_resource_ttl_ms: app_state.hls_proxy.transient_resource_ttl_ms(), + manifest_commit_requirement, + access_lease_id: Some(access_lease_id.clone()), + disabled_headers: app_state.get_disabled_headers(), + now_ms, + origin_io: Some(origin_io), + }; + if handoff_previous_rendered_at_ms.is_some() { + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + let _started = maybe_trigger_origin_refresh(refresh_request).await; + let strip = app_state.hls_proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + ) + .await + { + clear_hls_provisioning_handoff_consumer(app_state, origin.input, context.virtual_id, current_time_millis()); + return Some(response); + } + if let Some(response) = hls_post_refresh_access_lease_response_flag_manifest_response( + app_state, + path_proxy_session_id, + access_lease_id, + &context.username, + ) + .await + { + return Some(response); + } + return Some(hls_manifest_channel_unavailable_response_for_username(app_state, &context.username)); + } + match session_outcome { + HlsSessionStoreOutcome::Created => { + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + let _started = maybe_trigger_origin_refresh(refresh_request).await; + let strip = app_state.hls_proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + ) + .await + { + return Some(response); + } + } + HlsSessionStoreOutcome::Reused => { + let _started = maybe_trigger_origin_refresh(refresh_request).await; + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + let strip = app_state.hls_proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + ) + .await + { + return Some(response); + } + } + } + + if let Some(response) = hls_post_refresh_access_lease_response_flag_manifest_response( + app_state, + path_proxy_session_id, + access_lease_id, + &context.username, + ) + .await + { + return Some(response); + } + Some(hls_manifest_channel_unavailable_response_for_username(app_state, &context.username)) +} + +fn hls_initial_manifest_decision_wait_timeout() -> Duration { Duration::from_secs(90) } + +async fn hls_manifest_wait_timeout_for_requirement( + session: &HlsSessionHandle, + requirement: HlsManifestCommitRequirement, +) -> Duration { + match requirement { + HlsManifestCommitRequirement::FreshCommitRequired { .. } => hls_initial_manifest_decision_wait_timeout(), + HlsManifestCommitRequirement::CommittedManifestAllowed => hls_initial_manifest_wait_timeout(session).await, + } +} + +async fn touch_initial_manifest_access_lease_window( + app_state: &Arc, + access_lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + access_lease_state: HlsAccessLeaseState, + wait_timeout: Duration, + now_ms: u64, +) { + if wait_timeout.is_zero() || access_lease_state != HlsAccessLeaseState::Pending { + return; + } + let wait_timeout_ms = duration_to_millis_saturating(wait_timeout); + let deadline_ms = now_ms.saturating_add(wait_timeout_ms.max(hls_pending_bootstrap_window_ms())); + let _ = app_state + .hls_proxy + .touch_manifest_access_lease( + access_lease_id, + proxy_session_id, + now_ms, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms }), + hls_access_lease_ttl_ms(app_state), + ) + .await; +} + +async fn hls_initial_manifest_wait_timeout(session: &HlsSessionHandle) -> Duration { + let session = session.read().await; + if matches!( + session.account_binding_protection(current_time_millis()), + HlsAccountBindingProtection::NoMediaYet | HlsAccountBindingProtection::Expired + ) { + hls_initial_manifest_decision_wait_timeout() + } else { + Duration::ZERO + } +} + +async fn hls_segment_request_requires_origin_work(session: &HlsSessionHandle, segment_file: &HlsSegmentFile) -> bool { + let session = session.read().await; + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return false; + }; + if entry.proxy_file_ext != segment_file.extension { + return false; + } + matches!(entry.status, SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. }) + && entry.origin_fetch_ref.is_some() +} + +async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool { + let session = session.read().await; + session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) +} + +fn hls_transient_origin_binding_requires_runtime_prepare( + app_state: &Arc, + binding: &HlsOriginAccountBinding, +) -> bool { + binding.is_detached() + || (binding.is_active() + && matches!( + hls_origin_account_status(app_state, binding), + HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired + )) +} + +async fn prepare_hls_origin_binding_for_authorized_resource_work( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + req_headers: &HeaderMap, + work_kind: HlsOriginWorkKind, + now_ms: u64, +) -> Result, HlsOriginRuntimeAcquireError> { + if !hls_origin_binding_needs_reacquire(session).await { + return Ok(None); + } + if session.read().await.activity.active_origin_work_count > 0 { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) + .await + .map_err(HlsOriginRuntimeAcquireError::Fatal)?; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let origin_policy = hls_effective_origin_acquire_policy(session).await; + let prepared_origin = prepare_hls_origin_runtime( + app_state, + session, + &request_context.input, + &request_context.hls_url, + request_context.session_entry_url.as_str(), + &proxy_session_id, + fingerprint, + origin_policy.connection_kind, + origin_policy.priority, + work_kind, + HlsOriginWorkClass::Demand, + now_ms, + ) + .await + ?; + if let Some(binding) = prepared_origin.origin_account_binding_to_store { + session.write().await.replace_origin_account_binding(Some(binding)); + } + Ok(prepared_origin.preacquired_origin_account_handle) +} + +#[allow(clippy::too_many_lines)] +async fn prepare_hls_transient_origin_io_for_authorized_resource_work( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + req_headers: &HeaderMap, + now_ms: u64, +) -> Result, HlsOriginRuntimeAcquireError> { + let existing_binding = session.read().await.origin_account_binding.clone(); + let origin_policy = hls_effective_origin_acquire_policy(session).await; + let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await; + if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) { + match hls_origin_account_status(app_state, binding) { + HlsOriginAccountStatus::Known => { + let origin_io = HlsOriginIoContext { + app_state: Arc::clone(app_state), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::Demand.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + }; + let started_generation = session.write().await.start_origin_work(); + if let Ok(lease_guard) = begin_hls_origin_account_io_bounded( + &origin_io, + session, + binding, + hls_object_body_deadline(app_state.hls_proxy.segment_fetch_policy().origin_segment_timeout_ms), + ) + .await + { + return Ok(Some(HlsTransientOriginIoGuard::new( + Arc::clone(session), + origin_io, + lease_guard, + started_generation, + ))); + } + session.write().await.finish_origin_work(started_generation); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {} + } + } + + if !existing_binding + .as_ref() + .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(app_state, binding)) + { + return Ok(None); + } + if session.read().await.activity.active_origin_work_count > 0 { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) + .await + .map_err(HlsOriginRuntimeAcquireError::Fatal)?; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let prepared_origin = prepare_hls_origin_runtime( + app_state, + session, + &request_context.input, + &request_context.hls_url, + request_context.session_entry_url.as_str(), + &proxy_session_id, + fingerprint, + origin_policy.connection_kind, + origin_policy.priority, + HlsOriginWorkKind::Resource, + HlsOriginWorkClass::Demand, + now_ms, + ) + .await + ?; + if let Some(binding) = prepared_origin.origin_account_binding_to_store { + session.write().await.replace_origin_account_binding(Some(binding)); + } + let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle else { + return Ok(None); + }; + let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active) + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + let origin_io = HlsOriginIoContext { + app_state: Arc::clone(app_state), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::Demand.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + } + .with_preacquired_provider_handle(provider_handle); + let started_generation = session.write().await.start_origin_work(); + let Ok(lease_guard) = begin_hls_origin_account_io_bounded( + &origin_io, + session, + &binding, + hls_object_body_deadline(app_state.hls_proxy.segment_fetch_policy().origin_segment_timeout_ms), + ) + .await + else { + session.write().await.finish_origin_work(started_generation); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), origin_io, lease_guard, started_generation))) +} + +async fn try_hls_cached_manifest_response( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, + options: HlsCachedManifestOptions, +) -> Option { + let started_at = tokio::time::Instant::now(); + let started_at_ms = current_time_millis(); + loop { + let (transient_body, rendered_body, should_wait, wait_for_initial_commit) = { + let session = session.read().await; + let now_ms = current_time_millis(); + let should_wait = session.initial_manifest_commit_work_pending(); + let committed_body = hls_committed_manifest_body_for_request(&session, options, started_at_ms, now_ms); + let (transient_body, rendered_body) = match committed_body { + Some(HlsCommittedManifestBody::Transient(body)) => (Some(body), None), + Some(HlsCommittedManifestBody::Normal(body)) => (None, Some(body)), + None => (None, None), + }; + let wait_for_initial_commit = hls_should_wait_for_initial_manifest_commit( + &session, + transient_body.is_some() || rendered_body.is_some(), + should_wait, + options, + now_ms, + ); + (transient_body, rendered_body, should_wait, wait_for_initial_commit) + }; + if !wait_for_initial_commit { + if let Some(body) = transient_body { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let body = materialize_shared_hls_access_manifest( + &body, + &proxy_session_id, + access_lease_id, + access_lease_state, + strip, + "transient", + server_path, + ); + touch_pending_manifest_follow_up_window(app_state, session, access_lease_id, access_lease_state).await; + mark_successful_canonical_manifest_activity(app_state, session, current_time_millis()).await; + return Some(hls_response(body).into_response()); + } + if let Some(body) = rendered_body { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let body = materialize_shared_hls_access_manifest( + &body, + &proxy_session_id, + access_lease_id, + access_lease_state, + strip, + "normal", + server_path, + ); + touch_pending_manifest_follow_up_window(app_state, session, access_lease_id, access_lease_state).await; + mark_successful_canonical_manifest_activity(app_state, session, current_time_millis()).await; + return Some(hls_response(body).into_response()); + } + } + if options.wait_timeout.is_zero() || !should_wait || started_at.elapsed() >= options.wait_timeout { + return None; + } + + let remaining = options.wait_timeout.saturating_sub(started_at.elapsed()); + tokio::time::sleep(remaining.min(Duration::from_millis(25))).await; + } +} + +async fn mark_successful_canonical_manifest_activity( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) { + mark_hls_authorized_media_access(app_state, session, now_ms).await; +} + +async fn mark_hls_authorized_manifest_access(app_state: &Arc, session: &HlsSessionHandle, now_ms: u64) { + session.write().await.mark_authorized_manifest_access(now_ms); + app_state.hls_proxy.schedule_session_idle_for_handle(session).await; +} + +async fn mark_hls_authorized_media_access(app_state: &Arc, session: &HlsSessionHandle, now_ms: u64) { + session.write().await.mark_authorized_media_access(now_ms); + app_state.hls_proxy.schedule_session_idle_for_handle(session).await; +} + +fn hls_cache_configured(app_state: &Arc) -> bool { + let config = app_state.app_config.config.load(); + config.reverse_proxy.as_ref().is_some_and(|reverse_proxy| reverse_proxy.hls_cache.is_some()) +} + +fn hls_cache_enabled_for_target(app_state: &Arc, target: &ConfigTarget) -> bool { + hls_cache_configured(app_state) && is_hls_stream_share_enabled(target) +} + +pub(in crate::api) fn build_virtual_hls_entry_path( + target: &ConfigTarget, + input: &ConfigInput, + user: &ProxyUserCredentials, + virtual_id: u32, +) -> String { + if input.input_type.is_m3u() && !target.has_output(TargetType::Xtream) { + format!("/{}/live/{}/{}/{}{HLS_EXT}", storage_const::M3U_STREAM_PATH, user.username, user.password, virtual_id) + } else { + format!("/live/{}/{}/{}{HLS_EXT}", user.username, user.password, virtual_id) + } +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(in crate::api) async fn handle_hls_stream_request( fingerprint: &Fingerprint, app_state: &Arc, user: &ProxyUserCredentials, - target_id: u16, + target: &ConfigTarget, user_session: Option<&UserSession>, hls_url: &str, - archive_reference: Option, + _archive_reference: Option, virtual_id: u32, input: &ConfigInput, req_headers: &HeaderMap, connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, + connection_kind: Option, + original_hls_entry_path: &str, ) -> impl IntoResponse + Send { if app_state.active_users.is_user_blocked_for_stream(&user.username, virtual_id).await { return axum::http::StatusCode::BAD_REQUEST.into_response(); } + let stream_ref = hls_stream_ref_from_virtual_id(virtual_id); let normalized_hls_url = normalize_xtream_live_hls_url(hls_url, input); if normalized_hls_url != hls_url { debug_if_enabled!( @@ -197,14 +5217,48 @@ pub(in crate::api) async fn handle_hls_stream_request( sanitize_sensitive_info(&normalized_hls_url) ); } - let url = replace_url_extension(&normalized_hls_url, HLS_EXT); + let url = ensure_hls_manifest_extension(&normalized_hls_url); + let hls_cache_origin = build_hls_origin_resolution(input, &url); + let hls_origin_source = hls_cache_origin.as_ref().map(|_| build_hls_origin_source(input, stream_ref.clone())); let server_info = app_state.app_config.get_user_server_info(user); let Some(server_info) = server_info else { return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); }; - let hls_session_ttl_secs = get_hls_session_ttl_secs(app_state); - let (request_url, session_token, provider_handle) = if let Some(session) = user_session { + let disabled_headers = app_state.get_disabled_headers(); + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let headers = build_hls_manifest_request_headers( + &input.headers, + req_headers, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + + if hls_cache_enabled_for_target(app_state, target) { + let Some(origin_source) = hls_origin_source.clone() else { + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + if let Some(response) = try_hls_cache_entry_redirect( + app_state, + fingerprint, + user, + origin_source, + virtual_id, + user_session, + hls_cache_origin.as_ref().map_or(url.as_str(), |origin| origin.session_entry_url.as_str()), + input, + connection_permission, + connection_kind, + server_info.path.as_deref(), + ) + .await + { + return response; + } + } + + let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); + let (request_url, session_token, provider_handle, _selected_provider_config) = if let Some(session) = user_session { let pinned_provider = if session.provider.is_empty() { &input.name } else { &session.provider }; let provider_handle = if let Some(handle) = app_state .active_provider @@ -212,8 +5266,17 @@ pub(in crate::api) async fn handle_hls_stream_request( pinned_provider, &fingerprint.addr, false, - connection_priority_for_kind(user, session.connection_kind.unwrap_or(connection_kind)), - session.connection_kind.unwrap_or(connection_kind), + connection_priority_for_kind( + user, + session + .connection_kind + .or(connection_kind) + .unwrap_or(crate::api::model::ConnectionKind::Normal), + ), + session + .connection_kind + .or(connection_kind) + .unwrap_or(crate::api::model::ConnectionKind::Normal), Some(session.token.as_str()), ) .await @@ -229,12 +5292,21 @@ pub(in crate::api) async fn handle_hls_stream_request( }; if provider_handle.is_none() { - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; } - match provider_handle.as_ref().map(|handle| &handle.allocation) { - Some(ProviderAllocation::Exhausted) => (url, None, provider_handle), + Some(ProviderAllocation::Exhausted) => (url, None, provider_handle, None), Some(ProviderAllocation::Available(cfg) | ProviderAllocation::GracePeriod(cfg)) => { + let selected_provider_config = Arc::clone(cfg); let Some(stream_url) = get_stream_alternative_url(&url, input, cfg) else { app_state.connection_manager.release_provider_handle(provider_handle).await; return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); @@ -249,77 +5321,75 @@ pub(in crate::api) async fn handle_hls_stream_request( stream_url: &stream_url, addr: &fingerprint.addr, connection_permission, - connection_kind: session.connection_kind, + connection_kind: session.connection_kind.or(connection_kind), socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), }) .await; + let hls_session_ttl_secs = get_hls_session_ttl_secs(app_state); app_state .active_provider .refresh_provider_reservation(&cfg.name, &session_token, hls_session_ttl_secs) .await; - (stream_url, Some(session_token), provider_handle) + (stream_url, Some(session_token), provider_handle, Some(selected_provider_config)) } - None => (url, None, None), + None => (url, None, None, None), } } else { - let manifest_item_type = if archive_reference.is_some() { - PlaylistItemType::Catchup + let user_session_token = create_playback_session_fingerprint( + fingerprint, + &user.username, + virtual_id, + PlaylistItemType::LiveHls, + None, + ); + let hls_session_owner = if hls_cache_enabled_for_target(app_state, target) { + let session_key = HlsSessionKey::new(input.id, virtual_id.to_string()); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + Some(build_hls_origin_session_owner(&proxy_session_id)) } else { - PlaylistItemType::LiveHls + None }; - let user_session_token = - create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, manifest_item_type, None); - match app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &fingerprint.addr, - false, - connection_priority_for_kind(user, connection_kind), - connection_kind, - Some(&user_session_token), + let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); + let hls_session_ttl_secs = get_hls_session_ttl_secs(app_state); + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + input, + virtual_id, + &url, + &user_session_token, + session_owner, + hls_session_ttl_secs, + connection_permission, + fallback_connection_kind, + true, + ) + .await + else { + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, ) - .await - { - Some(provider_handle) => match provider_handle.allocation.get_provider_config() { - Some(provider_cfg) => { - let Some(stream_url) = get_stream_alternative_url(&url, input, &provider_cfg) else { - app_state - .connection_manager - .release_provider_handle(Some(provider_handle)) - .await; - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - }; - debug_if_enabled!( - "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", - sanitize_sensitive_info(&user.username), - provider_cfg.name, - sanitize_sensitive_info(&stream_url) - ); - let session_token = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token: &user_session_token, - virtual_id, - provider: &provider_cfg.name, - stream_url: &stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), - }) - .await; - app_state - .active_provider - .refresh_provider_reservation(&provider_cfg.name, &session_token, hls_session_ttl_secs) - .await; - (stream_url, Some(session_token), Some(provider_handle)) - } - None => (url, None, Some(provider_handle)), - }, - None => (url, None, None), - } + .await; + }; + debug_if_enabled!( + "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", + sanitize_sensitive_info(&user.username), + reservation.selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), + sanitize_sensitive_info(&reservation.request_url) + ); + ( + reservation.request_url, + Some(reservation.session_token), + reservation.provider_handle, + reservation.selected_provider_config, + ) }; // Playlist requests only need the chosen provider account to derive the URL and pin the session. @@ -327,13 +5397,6 @@ pub(in crate::api) async fn handle_hls_stream_request( // breaks forced same-account reuse on the next HLS/Catchup stream request. app_state.connection_manager.release_provider_handle(provider_handle).await; - // Don't forward Range on playlist fetch; segments use original headers in provider path - let filter_header: HeaderFilter = Some(Box::new(|name: &str| !name.eq_ignore_ascii_case("range"))); - let forwarded = get_headers_from_request(req_headers, &filter_header); - let disabled_headers = app_state.get_disabled_headers(); - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let headers = - request::get_request_headers(None, Some(&forwarded), disabled_headers.as_ref(), default_user_agent.as_deref()); let input_source = InputSource::from(input).with_url(request_url); let use_manual_redirects = app_state.should_use_manual_redirects(); let download_result = if use_manual_redirects { @@ -365,7 +5428,7 @@ pub(in crate::api) async fn handle_hls_stream_request( base_url: &base_url, content: &content, hls_url: response_url, - target_id, + target_id: target.id, virtual_id, input_id: input.id, user_token: session_token.as_deref(), @@ -379,7 +5442,8 @@ pub(in crate::api) async fn handle_hls_stream_request( .update_session_provider_headers(&user.username, session_token, &session_headers) .await; } - release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr).await; + release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) + .await; } hls_response(hls_content).into_response() } @@ -389,22 +5453,12 @@ pub(in crate::api) async fn handle_hls_stream_request( terminate_failed_hls_manifest_session(app_state, &user.username, session_token).await; } - let custom_stream_response = app_state.app_config.custom_stream_response.load(); - if custom_stream_response.as_ref().and_then(|c| c.channel_unavailable.as_ref()).is_some() { - let custom_stream_token = create_access_token(&app_state.app_config.access_token_secret, 30); - let url = format!( - "{}/{CUSTOM_VIDEO_PREFIX}/{}/{}/{}.ts?token={custom_stream_token}", - server_info.get_base_url(), - user.username, - user.password, - CustomVideoStreamType::ChannelUnavailable - ); - - let playlist = PLAYLIST_TEMPLATE.replace("{url}", &url); - hls_response(playlist).into_response() - } else { - axum::http::StatusCode::NOT_FOUND.into_response() - } + hls_custom_video_manifest_response( + app_state, + user, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) } } } @@ -423,6 +5477,36 @@ async fn get_stream_channel( m3u_get_item_for_stream_id(virtual_id, app_state, target).await.ok().map(|pli| pli.to_stream_channel(target_id)) } +pub(in crate::api) async fn resolve_hls_virtual_input_for_target( + app_state: &Arc, + target: &Arc, + virtual_id: u32, +) -> Option> { + let channel = get_stream_channel(app_state, target, virtual_id).await?; + app_state.app_config.get_input_by_name(&channel.input_name) +} + +async fn resolve_hls_origin_playlist_url( + app_state: &Arc, + target: &Arc, + input: &ConfigInput, + virtual_id: u32, + fallback_url: &str, +) -> Result { + if input.input_type.is_xtream() && target.has_output(TargetType::Xtream) { + let pli = xtream_get_item_for_stream_id(virtual_id, app_state, target, None) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; + let hls_extension = format!(".{HLS_EXT}"); + let (query_path, _) = get_query_path("", Some(&hls_extension), &pli, app_state); + return get_xtream_player_api_stream_url(input, ApiStreamContext::Live, &query_path, &pli.url) + .map(|url| url.to_string()) + .ok_or(StatusCode::SERVICE_UNAVAILABLE); + } + + Ok(fallback_url.to_string()) +} + async fn resolve_stream_channel( app_state: &Arc, target: &Arc, @@ -467,6 +5551,133 @@ async fn resolve_stream_channel( channel } +struct HlsAccessManifestRequestContext { + input: Arc, + hls_url: String, + session_entry_url: HlsOriginEntryUrl, + original_hls_entry_path: String, + origin_source: HlsOriginSource, + headers: HeaderMap, + server_path: Option, +} + +async fn resolve_hls_playback_manifest_request_context( + app_state: &Arc, + access_context: &HlsAccessContext, + req_headers: &HeaderMap, +) -> Result { + let Some((user, target)) = app_state.app_config.get_target_for_username(&access_context.username) else { + return Err(StatusCode::NOT_FOUND); + }; + if !hls_cache_enabled_for_target(app_state, &target) { + return Err(StatusCode::NOT_FOUND); + } + let Some(input) = app_state.app_config.get_input_by_id(access_context.input_id) else { + return Err(StatusCode::NOT_FOUND); + }; + if app_state.active_users.is_user_blocked_for_stream(&user.username, access_context.virtual_id).await { + return Err(StatusCode::FORBIDDEN); + } + let Some(channel) = get_stream_channel(app_state, &target, access_context.virtual_id).await else { + return Err(StatusCode::NOT_FOUND); + }; + let origin_playlist_url = + resolve_hls_origin_playlist_url(app_state, &target, &input, access_context.virtual_id, channel.url.as_ref()) + .await?; + let Some(hls_cache_origin) = build_hls_origin_resolution(&input, &origin_playlist_url) else { + return Err(StatusCode::SERVICE_UNAVAILABLE); + }; + let origin_source = build_hls_origin_source(&input, access_context.stream_ref.clone()); + let Some(server_info) = app_state.app_config.get_user_server_info(&user) else { + return Err(StatusCode::SERVICE_UNAVAILABLE); + }; + let disabled_headers = app_state.get_disabled_headers(); + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let headers = build_hls_manifest_request_headers( + &input.headers, + req_headers, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, access_context.virtual_id); + + Ok(HlsAccessManifestRequestContext { + input, + hls_url: hls_cache_origin.hls_url, + session_entry_url: hls_cache_origin.session_entry_url, + original_hls_entry_path, + origin_source, + headers, + server_path: server_info.path.clone(), + }) +} + +async fn hls_proxy_manifest( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id); + let now_ms = current_time_millis(); + let access_lease_snapshot = app_state + .hls_proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) + .await; + if let Some(session) = app_state.hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await { + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + now_ms, + ) + .await; + } + let (access_context, access_lease_state) = match hls_manifest_access_context_and_state( + &app_state, + &fingerprint, + &proxy_session_id, + &access_lease_id, + access_lease_snapshot.as_ref(), + now_ms, + ) + .await + { + Ok(context_and_state) => context_and_state, + Err(response) => return *response, + }; + let request_context = + match resolve_hls_playback_manifest_request_context(&app_state, &access_context, &headers).await { + Ok(context) => context, + Err(status) => return hls_canonical_status_response(status), + }; + + try_hls_cache_canonical_manifest_response( + &app_state, + &fingerprint, + &access_context, + &proxy_session_id, + &access_lease_id, + access_lease_state, + HlsCacheManifestOrigin { + raw_request_url: &request_context.hls_url, + session_entry_url: request_context.session_entry_url.clone(), + input: &request_context.input, + origin_source: request_context.origin_source, + }, + request_context.headers, + request_context.server_path.as_deref(), + &request_context.original_hls_entry_path, + ) + .await + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()) +} + #[allow(clippy::too_many_lines)] async fn hls_api_stream( fingerprint: Fingerprint, @@ -527,7 +5738,7 @@ async fn hls_api_stream_resolved( if user.permission_denied(&app_state) { let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, "", None).await; - return admission_failure_response( + return hls_admission_failure_manifest_response( &app_state, &fingerprint, &user, @@ -543,22 +5754,34 @@ async fn hls_api_stream_resolved( let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, &token) else { return axum::http::StatusCode::BAD_REQUEST.into_response(); }; + if hls_cache_enabled_for_target(&app_state, &target) { + return hls_custom_video_manifest_redirect_response_for_username( + &app_state, + &user.username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ); + } let lookup_session_token = decoded_hls_token .0 .clone() .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, virtual_id, false)); - let mut user_session = app_state - .active_users - .get_and_update_user_session(&user.username, &lookup_session_token) - .await; + let mut user_session = + app_state.active_users.get_and_update_user_session(&user.username, &lookup_session_token).await; if let Some(session) = &mut user_session { let decoded_archive_reference = m3u_archive_epg_reference_ts(&decoded_hls_token.1); if session.permission == UserConnectionPermission::Exhausted { - let stream_channel = - resolve_stream_channel(&app_state, &target, &input, virtual_id, &decoded_hls_token.1, decoded_archive_reference) - .await; - return admission_failure_response( + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &decoded_hls_token.1, + decoded_archive_reference, + ) + .await; + return hls_admission_failure_manifest_response( &app_state, &fingerprint, &user, @@ -570,10 +5793,16 @@ async fn hls_api_stream_resolved( } if app_state.active_provider.is_over_limit(&session.provider).await { - let stream_channel = - resolve_stream_channel(&app_state, &target, &input, virtual_id, &decoded_hls_token.1, decoded_archive_reference) - .await; - return admission_failure_response( + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &decoded_hls_token.1, + decoded_archive_reference, + ) + .await; + return hls_admission_failure_manifest_response( &app_state, &fingerprint, &user, @@ -593,11 +5822,9 @@ async fn hls_api_stream_resolved( let archive_reference = m3u_archive_epg_reference_ts(&hls_url); session.stream_url = hls_url.clone(); if session.virtual_id == virtual_id { - app_state - .connection_manager - .touch_http_activity(&user.username, &session.token, &fingerprint.addr) - .await; - let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; + app_state.connection_manager.touch_http_activity(&user.username, &session.token, &fingerprint.addr).await; + let stream_channel = + resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; if is_seek_request(stream_channel.cluster, &req_headers).await { // partial request means we are in reverse proxy mode, seek happened return force_provider_stream_response( @@ -641,21 +5868,19 @@ async fn hls_api_stream_resolved( ) .await; let connection_permission = connection_admission.permission; - let connection_kind = connection_admission - .kind - .or(session.connection_kind) - .unwrap_or(crate::api::model::ConnectionKind::Normal); + let connection_kind = connection_admission.kind.or(session.connection_kind); session.permission = connection_permission; - session.connection_kind = Some(connection_kind); - if connection_permission == UserConnectionPermission::Exhausted { - let provider = if session.provider.is_empty() { - input.name.clone() - } else { - session.provider.clone() - }; + if let Some(connection_kind) = connection_kind { + session.connection_kind = Some(connection_kind); + } + if connection_permission == UserConnectionPermission::Exhausted + || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) + { + let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; let stream_channel = - resolve_stream_channel(&app_state, &target, &input, virtual_id, &session.stream_url, archive_reference).await; - return admission_failure_response( + resolve_stream_channel(&app_state, &target, &input, virtual_id, &session.stream_url, archive_reference) + .await; + return hls_admission_failure_manifest_response( &app_state, &fingerprint, &user, @@ -665,13 +5890,15 @@ async fn hls_api_stream_resolved( ConnectFailureReason::UserConnectionsExhausted, ); } + let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); if is_hls_url(&session.stream_url) { + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); return handle_hls_stream_request( &fingerprint, &app_state, &user, - target.id, + &target, Some(session), &session.stream_url, archive_reference, @@ -680,13 +5907,15 @@ async fn hls_api_stream_resolved( &req_headers, connection_permission, connection_kind, + &original_hls_entry_path, ) .await .into_response(); } if is_file_url(&session.stream_url) { - let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; + let stream_channel = + resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; return local_stream_response( &fingerprint, &app_state, @@ -696,7 +5925,7 @@ async fn hls_api_stream_resolved( &target, &user, connection_permission, - connection_kind, + fallback_connection_kind, Some(&session.token), Some(request_class), false, @@ -705,7 +5934,8 @@ async fn hls_api_stream_resolved( .into_response(); } - let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; + let stream_channel = + resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; force_provider_stream_response( &fingerprint, &app_state, @@ -719,8 +5949,8 @@ async fn hls_api_stream_resolved( }, grace_mode, ) - .await - .into_response() + .await + .into_response() } else { axum::http::StatusCode::BAD_REQUEST.into_response() } @@ -728,15 +5958,82 @@ async fn hls_api_stream_resolved( pub fn hls_api_register() -> axum::Router> { axum::Router::new() - .route("/hls/{username}/{password}/{target_id}/{input_id}/{stream_id}/{token}", axum::routing::get(hls_api_stream)) + .route( + "/hls/shared/live/{proxy_session_id}/{hls_access_lease_id}/manifest.m3u8", + axum::routing::get(hls_proxy_manifest), + ) + .route( + "/hls/shared/live/{proxy_session_id}/{hls_access_lease_id}/{segment_file}", + axum::routing::get(hls_proxy_segment), + ) + .route( + "/hls/shared/live/{proxy_session_id}/{hls_access_lease_id}/map/{map_file}", + axum::routing::get(hls_proxy_map), + ) + .route( + "/hls/shared/live/{proxy_session_id}/{hls_access_lease_id}/r/{resource_file}", + axum::routing::get(hls_proxy_resource), + ) + .route( + "/hls/{username}/{password}/{target_id}/{input_id}/{stream_id}/{token}", + axum::routing::get(hls_api_stream), + ) //cfg.service(web::resource("/hls/{token}/{stream}").route(web::get().to(xtream_player_api_hls_stream))); //cfg.service(web::resource("/play/{token}/{type}").route(web::get().to(xtream_player_api_play_stream))); } #[cfg(test)] mod tests { - use super::{extract_hls_provider_session_headers, m3u_archive_epg_reference_ts}; - use axum::http::HeaderMap; + use super::{ + build_hls_manifest_request_headers, extract_hls_provider_session_headers, hls_api_register, + m3u_archive_epg_reference_ts, + }; + use crate::{ + api::model::{ + begin_hls_origin_account_io, build_hls_custom_video_manifest_body, build_proxy_session_id, + build_transient_resource_id, finish_hls_origin_account_io, ActiveProviderManager, ActiveUserManager, + AppState, CacheAccessState, CancelTokens, ConnectionKind, ConnectionManager, CreateUserSessionParams, + CustomVideoStreamType, EventManager, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, + HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, + HlsAccessLeaseTiming, HlsAccessLeaseValidationError, HlsEffectiveOriginAcquirePolicy, + HlsFreshManifestRequiredReason, HlsLifecycleEvent, HlsLifecycleEventKey, HlsManifestCommitRequirement, + HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginIoContext, + HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentFile, HlsSessionHandle, + HlsSession, HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, ManualPlaylistUpdateRequest, + MapCacheStatus, MapEntry, MetadataUpdateManager, OriginMapKey, OriginSegmentFetchRef, OriginSegmentKey, + PlaybackLifecycle, PlaylistStorageState, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, + ProxyMapId, ProxySessionId, RenderedManifest, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + SegmentFetchPriority, SharedStreamManager, TransientResourceKind, TransientResourceRef, TransportStreamBuffer, + UpdateGuard, UserSession, + }, + auth::Fingerprint, + model::{ + ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigProvider, ConfigTarget, HlsCacheConfig, + ConfigSource, CustomStreamResponse, ProcessTargets, ProxyUserCredentials, ReverseProxyConfig, + ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, TargetUser, + }, + processing::parser::hls::{ + origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + rewrite_hls, RewriteHlsProps, + }, + utils::GeoIp, + }; + use arc_swap::{ArcSwap, ArcSwapOption}; + use axum::{ + body::Body, + extract::ConnectInfo, + http::{header, HeaderMap, HeaderName, HeaderValue, Request, Response, StatusCode}, + response::IntoResponse, + }; + use http_body_util::BodyExt; + use shared::model::{ConfigPaths, ConfigProviderDto, ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, HlsCacheConfigDto, HlsSegmentRepairMode, HlsStripMode, InputType, PlaylistItemType, ProviderUrlSelectionPolicy, ReverseProxyConfigDto, StreamConfigDto, TargetOutputDto, UserConnectionPermission, XtreamTargetOutputDto}; + use std::{collections::HashMap, fmt::Write as _, net::SocketAddr, sync::Arc, time::Duration}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{mpsc, RwLock}, + }; + use tower::ServiceExt; #[test] fn archive_epg_reference_supports_query_and_path_formats() { @@ -773,4 +6070,6303 @@ mod tests { assert_eq!(session_headers.get("cookie").map(String::as_str), Some("sid=abc; pref=1")); } + + fn test_app_config() -> Arc { + let mut hls_user = ProxyUserCredentials::default(); + hls_user.username = "hls-user".to_string(); + hls_user.password = "hls-pass".to_string(); + hls_user.max_connections = 1; + let api_proxy = ApiProxyConfig { + server: vec![ApiProxyServerInfo { + name: "default".to_string(), + protocol: "https".to_string(), + host: "example.test".to_string(), + port: None, + timezone: "UTC".to_string(), + message: String::new(), + path: Some("iptv".to_string()), + }], + user: vec![TargetUser { target: "default".to_string(), credentials: vec![Arc::new(hls_user)] }], + ..Default::default() + }; + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + custom_stream_response_enabled: true, + ..Default::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig { + batch_files: vec![], + provider: vec![], + inputs: vec![], + sources: vec![], + templates: None, + })), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::from(Some(Arc::new(api_proxy)))), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), + }) + } + + fn hls_custom_video_test_user() -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + user.password = "secret".to_string(); + user + } + + #[allow(dead_code)] + fn test_hls_share_target(hls_enabled: bool) -> ConfigTarget { + ConfigTarget::from(&ConfigTargetDto { + id: 1, + name: "default".to_string(), + options: Some(ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: hls_enabled, mpeg_ts: false }, + ..Default::default() + }), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())], + ..Default::default() + }) + } + + #[allow(dead_code)] + fn test_hls_input() -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: "http://origin.example.com".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + } + } + + #[allow(dead_code)] + fn store_test_sources_with_target(app_state: &Arc, input: ConfigInput, target: ConfigTarget) { + let input = Arc::new(input); + app_state.app_config.sources.store(Arc::new(SourcesConfig { + batch_files: vec![], + provider: vec![], + inputs: vec![Arc::clone(&input)], + sources: vec![ConfigSource { + inputs: vec![Arc::clone(&input.name)], + targets: vec![Arc::new(target)], + }], + templates: None, + })); + } + + fn configure_default_test_server(app_state: &Arc) { + let users = app_state + .app_config + .api_proxy + .load_full() + .as_ref() + .map_or_else(Vec::new, |api_proxy| api_proxy.user.clone()); + app_state.app_config.api_proxy.store(Some(Arc::new(ApiProxyConfig { + server: vec![ApiProxyServerInfo { + name: "default".to_string(), + protocol: "http".to_string(), + host: "127.0.0.1".to_string(), + port: Some("8901".to_string()), + timezone: "UTC".to_string(), + message: String::new(), + path: None, + }], + user: users, + ..Default::default() + }))); + } + + #[test] + fn hls_custom_video_manifest_uses_twelve_endlist_segments_for_non_provisioning() { + let user = hls_custom_video_test_user(); + let manifest = build_hls_custom_video_manifest_body( + "https://example.test/iptv/", + &user, + CustomVideoStreamType::UserConnectionsExhausted, + 42_000, + None, + ); + + assert!(manifest.contains("#EXT-X-TARGETDURATION:10")); + assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(manifest.contains("#EXT-X-ENDLIST")); + assert!(manifest.contains("https://example.test/iptv/cvs/hls/viewer/secret/user_connections_exhausted.ts")); + assert_eq!(manifest.matches("#EXTINF:10.0,").count(), 12); + assert_eq!( + manifest.matches("https://example.test/iptv/cvs/hls/viewer/secret/user_connections_exhausted.ts").count(), + 12 + ); + } + + #[test] + fn hls_initial_manifest_decision_wait_timeout_is_ninety_seconds() { + assert_eq!(super::hls_initial_manifest_decision_wait_timeout(), Duration::from_secs(90)); + } + + #[tokio::test] + async fn hls_manifest_channel_unavailable_redirects_to_cvs_manifest() { + let app_state = test_app_state(); + enable_channel_unavailable_custom_response(&app_state); + + let response = super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + } + + #[tokio::test] + async fn hls_access_lease_channel_unavailable_flag_redirects_to_cvs_manifest() { + let app_state = test_app_state(); + enable_channel_unavailable_custom_response(&app_state); + let flag = HlsAccessLeaseResponseFlag::ChannelUnavailable { + reason: HlsAccessLeaseChannelUnavailableReason::OriginAccountUnavailable, + set_at_ms: 1, + }; + + let response = super::hls_access_lease_response_flag_manifest_response(&app_state, "hls-user", &flag); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + } + + #[tokio::test] + async fn hls_manifest_channel_unavailable_falls_back_to_not_found_when_custom_response_is_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + + let response = super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn hls_manifest_low_priority_preempted_redirects_to_cvs_manifest() { + let app_state = test_app_state(); + enable_low_priority_preempted_custom_response(&app_state); + + let response = + super::hls_manifest_low_priority_preempted_redirect_response_for_username(&app_state, "hls-user"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/low_priority_preempted.m3u8")); + } + + #[tokio::test] + async fn hls_manifest_low_priority_preempted_falls_back_to_service_unavailable_without_custom_response() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + + let response = + super::hls_manifest_low_priority_preempted_redirect_response_for_username(&app_state, "hls-user"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } + + #[test] + fn hls_custom_video_manifest_uses_endlist_for_session_or_lease_expired() { + let user = hls_custom_video_test_user(); + let manifest = build_hls_custom_video_manifest_body( + "https://example.test/iptv", + &user, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + 42_000, + None, + ); + + assert!(manifest.contains("#EXT-X-ENDLIST")); + assert!(manifest.contains( + "https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts" + )); + assert_eq!(manifest.matches("#EXTINF:10.0,").count(), 12); + assert_eq!( + manifest + .matches("https://example.test/iptv/cvs/hls/viewer/secret/hls_session_or_lease_expired.ts") + .count(), + 12 + ); + } + + #[test] + fn hls_custom_video_manifest_uses_live_six_segment_window_for_provisioning() { + let user = hls_custom_video_test_user(); + let manifest = build_hls_custom_video_manifest_body( + "https://example.test/iptv", + &user, + CustomVideoStreamType::Provisioning, + 123_456, + Some(80510), + ); + + assert!(manifest.contains("#EXT-X-TARGETDURATION:2")); + assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-INDEPENDENT-SEGMENTS\n")); + assert!(!manifest.contains("#EXT-X-DISCONTINUITY-SEQUENCE")); + assert!(!manifest.contains("#EXT-X-SESSION-DATA")); + assert!(!manifest.contains("#EXT-X-ENDLIST")); + assert!(!manifest.contains("#EXT-X-DISCONTINUITY\n")); + assert_eq!(manifest.matches("#EXTINF:2.000000,").count(), 6); + for index in 0..6 { + assert!(manifest + .contains(&format!("https://example.test/iptv/cvs/hls/viewer/secret/provisioning_{index:03}.ts"))); + } + assert!(!manifest.contains("https://example.test/iptv/cvs/hls/viewer/secret/provisioning.ts")); + assert_eq!( + crate::api::model::hls_panel_provisioning_manifest_path(&user, 80510), + "/cvs/hls/viewer/secret/provisioning.m3u8?id=80510" + ); + assert!(!manifest.contains("provisioning.ts?")); + assert!(!manifest.contains("virtual_id")); + assert!(!manifest.contains("&seq=")); + } + + #[test] + fn hls_response_uses_rfc8216_manifest_content_type() { + let response = super::hls_response("#EXTM3U\n".to_string()).into_response(); + + assert_eq!(response.headers().get(header::CONTENT_TYPE).unwrap(), "application/vnd.apple.mpegurl"); + } + + #[test] + fn virtual_hls_entry_path_uses_single_manifest_extension() { + let user = hls_custom_video_test_user(); + let xtream_target = ConfigTarget::from(&shared::model::ConfigTargetDto { + output: vec![shared::model::TargetOutputDto::Xtream(shared::model::XtreamTargetOutputDto::default())], + ..Default::default() + }); + let xtream_input = ConfigInput { input_type: InputType::Xtream, ..ConfigInput::default() }; + let m3u_target = ConfigTarget::from(&shared::model::ConfigTargetDto::default()); + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; + + let xtream_path = super::build_virtual_hls_entry_path(&xtream_target, &xtream_input, &user, 59); + let m3u_path = super::build_virtual_hls_entry_path(&m3u_target, &m3u_input, &user, 59); + + assert_eq!(xtream_path, "/live/viewer/secret/59.m3u8"); + assert_eq!(m3u_path, "/m3u-stream/live/viewer/secret/59.m3u8"); + assert!(!xtream_path.contains("..m3u8")); + assert!(!m3u_path.contains("..m3u8")); + } + + #[test] + fn hls_manifest_headers_apply_disabled_headers_and_default_user_agent_policy() { + let mut input_headers = HashMap::new(); + input_headers.insert("User-Agent".to_string(), "Input-UA".to_string()); + input_headers.insert("Accept-Language".to_string(), "de".to_string()); + input_headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); + input_headers.insert("X-Origin-Secret".to_string(), "input-secret".to_string()); + + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + request_headers.insert(header::USER_AGENT, HeaderValue::from_static("Client-UA")); + request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer client-secret")); + request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + request_headers + .insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + request_headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("client")); + request_headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); + + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: true, + cloudflare_header: true, + custom_header: vec!["X-Origin-Secret".to_string()], + }; + let headers = + build_hls_manifest_request_headers(&input_headers, &request_headers, Some(&disabled), Some("Default-UA")); + + assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Input-UA"); + assert_eq!(headers.get("accept-language").expect("accept language"), "de"); + assert!(!headers.contains_key(header::RANGE)); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key("proxy-authorization")); + assert!(!headers.contains_key(header::HOST)); + assert!(!headers.contains_key("x-origin-secret")); + assert!(!headers.contains_key("x-blocked")); + assert!(!headers.contains_key("cf-ray")); + } + + #[test] + fn hls_proxy_public_path_prefix_rewrites_only_proxy_hls_uri_surfaces() { + let body = concat!( + "#EXTM3U\n", + "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/r/key.key\",IV=0x1\n", + "#EXT-X-MAP:URI=\"/hls/shared/live/proxy-id/map/000000.mp4\",BYTERANGE=\"10@0\"\n", + "#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/r/part.m4s\"\n", + "#EXT-X-MEDIA-SEQUENCE:7\n", + "#EXTINF:4.0,\n", + "/hls/shared/live/proxy-id/000007.ts\n", + "https://origin.example.com/not-proxy.ts\n", + ); + + let prefixed = super::apply_hls_proxy_public_path_prefix(body.to_string(), Some("/iptv/")); + + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/key.key\"")); + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/map/000000.mp4\"")); + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/part.m4s\"")); + assert!(prefixed.contains("\n/iptv/hls/shared/live/proxy-id/000007.ts\n")); + assert!(prefixed.contains("#EXT-X-MEDIA-SEQUENCE:7")); + assert!(prefixed.contains("https://origin.example.com/not-proxy.ts")); + } + + #[test] + fn hls_proxy_public_path_prefix_keeps_body_unchanged_without_server_path() { + let body = "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/000007.ts\n".to_string(); + + assert_eq!(super::apply_hls_proxy_public_path_prefix(body.clone(), None), body); + assert_eq!(super::apply_hls_proxy_public_path_prefix(body.clone(), Some("/")), body); + } + + #[test] + fn hls_manifest_materialization_uses_proxy_paths_without_provider_or_legacy_route() { + let body = format!( + "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/{}/000123.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + let lease_id = crate::api::model::HlsAccessLeaseId("access-lease".to_string()); + + let materialized = super::materialize_hls_access_manifest(&body, &lease_id, Some("/iptv")); + + assert!(materialized.contains("/iptv/hls/shared/live/proxy-id/access-lease/000123.ts")); + assert!(!materialized.contains("provider://")); + assert!(!materialized.contains("/hls/hls-user/")); + assert!(!materialized.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + } + + #[test] + fn hls_cache_origin_entry_url_preserves_provider_scheme_as_failover() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://origin.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { provider_configs: Some(vec![Arc::clone(&provider)]), ..ConfigInput::default() }; + + let origin = + super::resolve_hls_cache_origin_entry_url(&input, "provider://demo/live/account-a/token-a/1025130.m3u8") + .expect("provider entry url should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "provider://demo/live/account-a/token-a/1025130.m3u8"); + assert_eq!(origin.session_entry_url.url_failover_provider().expect("provider failover config").name.as_ref(), "demo"); + let provider_key = super::build_hls_origin_source(&input, "1025130").session_key(); + let direct_key = super::build_hls_origin_source(&input, "1025130").session_key(); + assert_eq!(provider_key, direct_key); + assert_eq!(provider_key.stable_value(), "input:0|hls|1025130"); + assert!(!provider_key.stable_value().contains("provider://")); + assert!(!provider_key.stable_value().contains("origin.example.com")); + } + + #[test] + fn hls_cache_origin_entry_url_does_not_attach_url_failover_provider_to_http_url() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://origin.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { provider_configs: Some(vec![provider]), ..ConfigInput::default() }; + + let origin = super::resolve_hls_cache_origin_entry_url( + &input, + "http://origin.example.com/live/account-a/token-a/1025130.m3u8", + ) + .expect("http entry url should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "http://origin.example.com/live/account-a/token-a/1025130.m3u8"); + assert!(origin.session_entry_url.url_failover_provider().is_none()); + } + + #[test] + fn hls_origin_source_kind_covers_xtream_m3u_and_direct_media_playlist() { + assert_eq!( + super::hls_origin_source_kind(InputType::Xtream), + crate::api::model::HlsOriginSourceKind::XtreamLive + ); + assert_eq!( + super::hls_origin_source_kind(InputType::M3u), + crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist + ); + assert_eq!( + super::hls_origin_source_kind(InputType::Library), + crate::api::model::HlsOriginSourceKind::DirectMediaPlaylist + ); + } + + #[test] + fn hls_manifest_extension_helper_does_not_create_double_dot_urls() { + assert_eq!( + super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123.m3u8"), + "http://origin.example.com/live/user/pass/1025123.m3u8" + ); + assert_eq!( + super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..m3u8"), + "http://origin.example.com/live/user/pass/1025123.m3u8" + ); + assert_eq!( + super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..?token=1"), + "http://origin.example.com/live/user/pass/1025123.m3u8?token=1" + ); + assert_eq!( + super::ensure_hls_manifest_extension("provider://mirror/live/user/pass/1025123..m3u8"), + "provider://mirror/live/user/pass/1025123.m3u8" + ); + } + + #[test] + fn hls_origin_resolution_preserves_legacy_built_xtream_origin_url() { + let input = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "http://origin.example.com/base".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + ..ConfigInput::default() + }; + + let origin = + super::build_hls_origin_resolution(&input, "http://other.example.com/live/other/creds/1025126.m3u8") + .expect("xtream origin should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "http://other.example.com/live/other/creds/1025126.m3u8"); + assert_eq!(origin.hls_url, origin.session_entry_url.as_str()); + assert!(origin.session_entry_url.url_failover_provider().is_none()); + } + + #[test] + fn hls_origin_resolution_keeps_provider_failover_out_of_identity() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "provider://mirror-group".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let origin = super::build_hls_origin_resolution( + &input, + "provider://mirror-group/live/source-user/source-pass/1025126.m3u8", + ) + .expect("provider failover origin should resolve"); + let failover_key = super::build_hls_origin_source(&input, "80510").session_key(); + let direct_key = super::build_hls_origin_source(&input, "80510").session_key(); + + assert_eq!(origin.session_entry_url.as_str(), "provider://mirror-group/live/source-user/source-pass/1025126.m3u8"); + assert!(origin.session_entry_url.url_failover_provider().is_some()); + assert_eq!(failover_key, direct_key); + assert!(!failover_key.stable_value().contains("provider://")); + assert!(!failover_key.stable_value().contains("mirror-a.example.com")); + assert!(!failover_key.stable_value().contains("mirror-b.example.com")); + } + + #[test] + fn hls_origin_resolution_uses_m3u_playlist_item_url() { + let input = ConfigInput { + id: 9, + name: Arc::from("m3u"), + input_type: InputType::M3u, + url: "http://playlist.example.com/list.m3u".to_string(), + ..ConfigInput::default() + }; + + let origin = super::build_hls_origin_resolution(&input, "http://media.example.com/live/channel/index.m3u8") + .expect("m3u hls origin should resolve"); + let source = super::build_hls_origin_source(&input, "stable-item"); + + assert_eq!(origin.session_entry_url.as_str(), "http://media.example.com/live/channel/index.m3u8"); + assert_eq!(source.source_kind, crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(source.session_key().stable_value(), "input:9|hls|stable-item"); + } + + #[test] + fn provider_failover_mirror_change_keeps_same_hls_session_identity() { + let provider_a = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-a.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let provider_b = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input_a = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "provider://mirror-group".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + provider_configs: Some(vec![provider_a]), + ..ConfigInput::default() + }; + let input_b = ConfigInput { provider_configs: Some(vec![provider_b]), ..input_a.clone() }; + + let origin_a = super::build_hls_origin_resolution(&input_a, "provider://mirror-group/a.m3u8") + .expect("provider failover origin a should resolve"); + let origin_b = super::build_hls_origin_resolution(&input_b, "provider://mirror-group/b.m3u8") + .expect("provider failover origin b should resolve"); + let key_a = super::build_hls_origin_source(&input_a, "80510").session_key(); + let key_b = super::build_hls_origin_source(&input_b, "80510").session_key(); + let secret = b"rewrite-secret"; + + assert!(origin_a.session_entry_url.url_failover_provider().is_some()); + assert!(origin_b.session_entry_url.url_failover_provider().is_some()); + assert_eq!(key_a, key_b); + assert_eq!(key_a.stable_value(), "input:7|hls|80510"); + assert_eq!(build_proxy_session_id(&key_a, secret), build_proxy_session_id(&key_b, secret)); + assert!(!key_a.stable_value().contains("provider://")); + assert!(!key_a.stable_value().contains("mirror-a.example.com")); + assert!(!key_a.stable_value().contains("mirror-b.example.com")); + } + + #[test] + fn hls_runtime_origin_fetch_url_uses_selected_provider_account() { + let input = ConfigInput { + name: Arc::from("source"), + url: "http://source.example.com".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let provider_input = ConfigInput { + id: 7, + name: Arc::from("selected-provider"), + url: "http://provider.example.com".to_string(), + username: Some("provider-user".to_string()), + password: Some("provider-pass".to_string()), + input_type: InputType::Xtream, + max_connections: 1, + ..ConfigInput::default() + }; + let provider = Arc::new(RuntimeProviderConfig::new( + &provider_input, + Arc::new(tokio::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + )); + + let fetch_url = super::build_hls_origin_fetch_url( + &input, + "http://source.example.com/live/source-user/source-pass/12345.m3u8", + "http://source.example.com/live/source-user/source-pass/12345.m3u8", + Some(&provider), + ) + .expect("fetch url should be rewritten"); + + assert_eq!(fetch_url, "http://provider.example.com/live/provider-user/provider-pass/12345.m3u8"); + + let failover_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let provider_scheme_input = ConfigInput { + name: Arc::from("source"), + url: "provider://demo".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + provider_configs: Some(vec![Arc::clone(&failover_provider)]), + ..ConfigInput::default() + }; + let provider_scheme_without_account_rewrite = super::build_hls_origin_fetch_url( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + None, + ) + .expect("provider scheme fetch url should remain failover capable"); + assert_eq!( + provider_scheme_without_account_rewrite, + "provider://demo/live/source-user/source-pass/12345.m3u8" + ); + + let provider_scheme_fetch_url = super::build_hls_origin_fetch_url( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + Some(&provider), + ) + .expect("provider scheme fetch url should use selected runtime account without losing failover"); + + assert_eq!( + provider_scheme_fetch_url, + "provider://demo/live/provider-user/provider-pass/12345.m3u8" + ); + let failover_context = super::hls_url_failover_provider_for_origin_context( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + &provider_scheme_fetch_url, + ) + .expect("provider failover context"); + assert_eq!(failover_context.name.as_ref(), "demo"); + } + + #[test] + fn hls_origin_entry_attaches_url_failover_provider_only_to_provider_scheme_fetch_url() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://mirror.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + + let http_provider = super::effective_hls_url_failover_provider_for_fetch_url( + "http://provider.example.com/live/user/pass/12345.m3u8", + None, + Some(Arc::clone(&provider)), + ); + let http_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( + "http://provider.example.com/live/user/pass/12345.m3u8", + http_provider, + ) + .expect("http origin entry"); + assert!(http_entry.url_failover_provider().is_none()); + + let provider_scheme_provider = super::effective_hls_url_failover_provider_for_fetch_url( + "provider://demo/live/user/pass/12345.m3u8", + None, + Some(Arc::clone(&provider)), + ); + let provider_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + provider_scheme_provider, + ) + .expect("provider origin entry"); + assert_eq!(provider_entry.url_failover_provider().expect("provider").name.as_ref(), "demo"); + } + + #[tokio::test] + async fn hls_access_lease_validity_uses_session_idle_timeout_not_cache_duration() { + let hls_dto = HlsCacheConfigDto { cache_duration: 900, session_idle_timeout: 42, ..Default::default() }; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + + assert_eq!(super::hls_access_lease_ttl_ms(&app_state), 42_000); + } + + #[tokio::test] + async fn hls_access_lease_active_window_uses_two_target_durations() { + let app_state = test_app_state(); + let key = HlsSessionKey::new(1, "access-window-stream"); + let (session, _) = + app_state.hls_proxy.get_or_create_session_with_outcome(key, b"secret", super::current_time_millis()).await; + session.write().await.target_duration = Some(11); + + let timing = super::hls_access_lease_timing_for_session(&app_state, &session).await; + + assert_eq!(timing.active_window_ms, 22_000); + assert_eq!(timing.valid_window_ms, super::hls_access_lease_ttl_ms(&app_state)); + } + + #[tokio::test] + async fn hls_lifecycle_active_timer_moves_access_lease_to_idle() { + let app_state = test_app_state(); + let now_ms = super::current_time_millis(); + let lease_id = HlsAccessLeaseId("lifecycle-lease".to_string()); + let key = HlsSessionKey::new(1, "lifecycle-stream"); + let (session, _) = app_state.hls_proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "lifecycle-stream".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + session.write().await.activity.active_access_lease_count = 1; + + app_state + .hls_proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseActive { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls_proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + Some(HlsAccessLeaseState::Idle) + ); + assert_eq!(session.read().await.activity.active_access_lease_count, 0); + } + + #[tokio::test] + async fn hls_lifecycle_validity_timer_removes_expired_access_lease() { + let mut hls_cache = HlsCacheConfigDto::default(); + hls_cache.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_cache); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::current_time_millis(); + let proxy_session_id = ProxySessionId("lifecycle-validity-proxy".to_string()); + let lease_id = HlsAccessLeaseId("lifecycle-validity-lease".to_string()); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "lifecycle-stream".to_string(), + 123, + now_ms, + 1, + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 1 }, + ) + .await + .is_activated()); + let repair_before = app_state.hls_proxy.segment_repair().stats().await; + assert_eq!(repair_before.windows, 1); + assert_eq!(repair_before.generations, 1); + + app_state + .hls_proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls_proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + None + ); + let repair_after = app_state.hls_proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); + } + + #[tokio::test] + async fn hls_lifecycle_validity_timer_removes_expired_pending_access_lease() { + let app_state = test_app_state(); + let now_ms = super::current_time_millis(); + let key = HlsSessionKey::new(1, "pending-expiry-stream"); + let (session, _) = app_state.hls_proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("pending-expiry-lease".to_string()); + let active_lease_id = HlsAccessLeaseId("active-soft-lease".to_string()); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "pending-expiry-stream".to_string(), + 123, + now_ms, + 1, + )) + .await; + app_state + .hls_proxy + .prepare_access_lease( + HlsAccessLease::pending( + active_lease_id.clone(), + HlsPlaybackFamilyKey::new("soft-user", "client"), + proxy_session_id.clone(), + "soft-user".to_string(), + "soft-session-token".to_string(), + 1, + "pending-expiry-stream".to_string(), + 123, + now_ms, + 60_000, + ) + .with_origin_acquire_policy(ConnectionKind::Soft, 20), + ) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &active_lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + + app_state + .hls_proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls_proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + None + ); + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 1); + let effective_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(effective_policy.connection_kind, ConnectionKind::Soft); + assert_eq!(effective_policy.priority, 20); + } + + #[tokio::test] + async fn hls_lifecycle_session_idle_timer_removes_idle_session() { + let mut hls_dto = HlsCacheConfigDto { session_idle_timeout: 1, ..Default::default() }; + hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::current_time_millis(); + let key = HlsSessionKey::new(1, "expired-session"); + let (session, _) = + app_state.hls_proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("session-idle-cleanup-lease".to_string()); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "expired-session".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 1); + assert_eq!(app_state.hls_proxy.segment_repair().stats().await.windows, 1); + + app_state + .hls_proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + due_at_ms: now_ms.saturating_sub(1_000), + }, + now_ms, + ) + .await; + + assert!(app_state.hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); + assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 0); + let repair_after = app_state.hls_proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); + } + + #[tokio::test] + async fn hls_gc_session_removal_cleans_access_leases_and_repair_state() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let mut hls_dto = HlsCacheConfigDto { + cache_path: Some(temp_dir.path().to_string_lossy().into_owned()), + session_idle_timeout: 1, + ..Default::default() + }; + hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::current_time_millis(); + let key = HlsSessionKey::new(1, "gc-cleanup-session"); + let (session, _) = + app_state.hls_proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("gc-cleanup-lease".to_string()); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "gc-cleanup-session".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 1); + assert_eq!(app_state.hls_proxy.segment_repair().stats().await.windows, 1); + + let report = app_state.hls_proxy.run_garbage_collection_once(now_ms).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 1); + assert!(app_state.hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); + assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 0); + let repair_after = app_state.hls_proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); + } + + fn test_app_state() -> Arc { test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::new())) } + + fn disable_custom_stream_response(app_state: &Arc) { + app_state.app_config.config.store(Arc::new(Config { + custom_stream_response_enabled: false, + ..Default::default() + })); + } + + fn enable_provider_exhausted_custom_response(app_state: &Arc) { + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: Some(TransportStreamBuffer::new(ts_packet)), + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); + } + + fn enable_low_priority_preempted_custom_response(app_state: &Arc) { + let config = app_state.app_config.config.load(); + app_state.app_config.config.store(Arc::new(Config { + custom_stream_response_enabled: true, + ..config.as_ref().clone() + })); + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: Some(TransportStreamBuffer::new(ts_packet)), + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); + } + + fn enable_channel_unavailable_custom_response(app_state: &Arc) { + let config = app_state.app_config.config.load(); + app_state.app_config.config.store(Arc::new(Config { + custom_stream_response_enabled: true, + ..config.as_ref().clone() + })); + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: Some(TransportStreamBuffer::new(ts_packet)), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); + } + + fn enable_hls_provisioning_custom_response(app_state: &Arc) { + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + let provisioning_segments = (0..6) + .map(|index| { + let mut packet = ts_packet.clone(); + packet[1] = u8::try_from(index).expect("test index fits"); + TransportStreamBuffer::new(packet) + }) + .collect(); + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: provisioning_segments, + }))); + } + + fn test_app_state_with_inputs(inputs: Vec>) -> Arc { + test_app_state_with_hls_proxy_and_inputs(Arc::new(HlsProxyManager::new()), inputs) + } + + fn enable_hls_cache(app_state: &Arc) { + let config = Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + })), + ..Default::default() + }; + app_state.app_config.config.store(Arc::new(config)); + } + + fn test_app_state_with_hls_proxy(hls_proxy: Arc) -> Arc { + test_app_state_with_hls_proxy_and_inputs(hls_proxy, Vec::new()) + } + + fn test_app_state_with_hls_proxy_and_inputs( + hls_proxy: Arc, + inputs: Vec>, + ) -> Arc { + let app_config = test_app_config(); + if !inputs.is_empty() { + app_config.sources.store(Arc::new(SourcesConfig { + batch_files: vec![], + provider: vec![], + inputs, + sources: vec![], + templates: None, + })); + } + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config = app_config.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + let connection_manager = Arc::new(ConnectionManager::new( + &active_users, + &active_provider, + &shared_stream_manager, + &event_manager, + None, + )); + let cancel_tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(cancel_tokens.metadata.clone())); + let (manual_update_sender, _) = mpsc::channel::(1); + + Arc::new(AppState { + forced_targets: Arc::new(ArcSwap::from_pointee(ProcessTargets { + enabled: false, + inputs: Vec::new(), + targets: Vec::new(), + target_names: Vec::new(), + })), + app_config, + http_client: Arc::new(ArcSwap::from_pointee(reqwest::Client::new())), + http_client_no_redirect: Arc::new(ArcSwap::from_pointee( + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + )), + downloads: Arc::new(crate::api::model::DownloadQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls_proxy, + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: Arc::new(ArcSwap::from_pointee(cancel_tokens)), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + update_guard: UpdateGuard::new(), + metadata_manager, + manual_update_sender, + }) + } + + async fn create_bound_hls_test_session( + app_state: &Arc, + input: &ConfigInput, + stream_ref: &str, + account_name: &str, + now_ms: u64, + ) -> HlsSessionHandle { + let origin_source = super::build_hls_origin_source(input, stream_ref); + let session_key = origin_source.session_key(); + let (session, _) = app_state + .hls_proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin_source, + &app_state.get_encrypt_secret(), + now_ms, + ) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + { + let mut session_guard = session.write().await; + session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from(account_name), + &proxy_session_id, + now_ms, + )); + } + session + } + + async fn create_unbound_hls_test_session( + app_state: &Arc, + input: &ConfigInput, + stream_ref: &str, + now_ms: u64, + ) -> HlsSessionHandle { + let origin_source = super::build_hls_origin_source(input, stream_ref); + let session_key = origin_source.session_key(); + app_state + .hls_proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin_source, + &app_state.get_encrypt_secret(), + now_ms, + ) + .await + .0 + } + + fn test_hls_origin_io_context(app_state: &Arc) -> HlsOriginIoContext { + HlsOriginIoContext { + app_state: Arc::clone(app_state), + client_addr: test_fingerprint().addr, + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + } + } + + fn single_hls_provider_input(name: &str) -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from(name), + input_type: InputType::Xtream, + url: "http://account.example.com".to_string(), + username: Some("account-user".to_string()), + password: Some("account-pass".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + } + } + + fn overlap_provider_input() -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from("overlap-input"), + input_type: InputType::Xtream, + url: "http://root.example.com".to_string(), + username: Some("root-user".to_string()), + password: Some("root-pass".to_string()), + enabled: true, + priority: 10, + max_connections: 1, + aliases: Some(vec![crate::model::ConfigInputAlias { + id: 2, + name: Arc::from("account-a"), + url: "http://account.example.com".to_string(), + username: Some("account-user".to_string()), + password: Some("account-pass".to_string()), + priority: 0, + max_connections: 1, + exp_date: None, + enabled: true, + }]), + ..ConfigInput::default() + } + } + + #[tokio::test] + async fn hls_origin_account_io_lease_allows_parallel_same_session_origin_work() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); + let origin_io = test_hls_origin_io_context(&app_state); + + let first = begin_hls_origin_account_io(&origin_io, &session, &binding) + .await + .expect("first same-session origin io acquires provider account"); + wait_for_provider_connection_count(&app_state, 1).await; + let second = begin_hls_origin_account_io(&origin_io, &session, &binding) + .await + .expect("second same-session origin io joins session lease"); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!( + session + .read() + .await + .origin_account_io_lease + .as_ref() + .expect("session provider lease exists") + .active_io_count, + 2 + ); + + finish_hls_origin_account_io(&origin_io, &session, first, true).await; + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!( + session + .read() + .await + .origin_account_io_lease + .as_ref() + .expect("session provider lease remains while second io is active") + .active_io_count, + 1 + ); + + finish_hls_origin_account_io(&origin_io, &session, second, true).await; + wait_for_provider_connection_count(&app_state, 0).await; + assert!( + app_state + .active_provider + .is_provider_reserved_for_other_session(&binding.account_name, Some("other-hls-session")) + .await + ); + } + + #[tokio::test] + async fn hls_origin_account_io_lease_blocks_other_hls_sessions_for_same_account() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let first_session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let second_session = create_bound_hls_test_session(&app_state, &input, "67890", "account-a", 1_000).await; + let first_binding = first_session.read().await.origin_account_binding.clone().expect("binding exists"); + let second_binding = second_session.read().await.origin_account_binding.clone().expect("binding exists"); + let origin_io = test_hls_origin_io_context(&app_state); + + let first = begin_hls_origin_account_io(&origin_io, &first_session, &first_binding) + .await + .expect("first session acquires account"); + wait_for_provider_connection_count(&app_state, 1).await; + + assert!(begin_hls_origin_account_io(&origin_io, &second_session, &second_binding).await.is_err()); + wait_for_provider_connection_count(&app_state, 1).await; + + finish_hls_origin_account_io(&origin_io, &first_session, first, true).await; + wait_for_provider_connection_count(&app_state, 0).await; + } + + #[tokio::test] + async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() { + let input = single_hls_provider_input("known-account"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let proxy_session_id = ProxySessionId("shared-hls-session".to_string()); + let known_binding = + HlsOriginAccountBinding::new(Arc::clone(&input.name), Arc::clone(&input.name), &proxy_session_id, 1_000); + let missing_binding = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("removed-account"), + &proxy_session_id, + 1_000, + ); + let mut detached_binding = known_binding.clone(); + detached_binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, 2_000); + + assert!(!super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &known_binding)); + assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &missing_binding)); + assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&app_state, &detached_binding)); + } + + #[tokio::test] + async fn hls_account_overlap_selects_soft_candidate_but_not_hard_active() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let session = create_bound_hls_test_session(&app_state, &input, "old", "account-a", 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + } + let new_proxy_session_id = ProxySessionId("new-session".to_string()); + + let hard_candidate = + super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 5_000).await; + assert!(hard_candidate.is_none(), "hard-active sessions must not be overbooked"); + + let delayed_candidate = + super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 12_000).await; + assert!(delayed_candidate.is_none(), "soft-active candidate must respect the dynamic overlap delay"); + + let soft_candidate = + super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 21_000) + .await + .expect("soft-active session can be overbooked"); + assert_eq!(soft_candidate.account_name.as_ref(), "account-a"); + assert_eq!(soft_candidate.last_media_at_ms, 1_000); + assert_eq!(soft_candidate.soft_overlap_eligible_at_ms, 21_000); + assert_eq!(soft_candidate.soft_overlap_delay_ms, 20_000); + assert_eq!(soft_candidate.reclaim_until_ms, 31_000); + } + + #[test] + fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() { + assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 1, 1), 20_000); + assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 3, 2), 15_000); + assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 4, 2), 10_000); + } + + #[tokio::test] + async fn hls_account_overlap_reclaim_preempts_speculative_session() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await; + let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await; + let winner_proxy_session_id = winner.read().await.proxy_session_id.clone(); + let loser_proxy_session_id = loser.read().await.proxy_session_id.clone(); + { + let mut loser = loser.write().await; + loser.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( + Arc::clone(&input.name), + Arc::from("account-a"), + &loser_proxy_session_id, + winner_proxy_session_id.clone(), + 20_000, + 2_000, + )); + } + let loser_generation = loser.read().await.activity.origin_work_generation; + + super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; + + assert!(app_state + .hls_proxy + .is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000) + .await); + assert!(!app_state + .hls_proxy + .is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000) + .await); + let loser_binding_mode = loser.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!( + loser_binding_mode, + HlsOriginAccountBindingMode::Detached { + reason: HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, + .. + } + )); + assert_eq!(loser.read().await.activity.origin_work_generation, loser_generation + 1); + let winner_binding_mode = winner.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!(winner_binding_mode, HlsOriginAccountBindingMode::Active)); + } + + #[tokio::test] + async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await; + let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await; + let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone(); + let promoted_proxy_session_id = promoted.read().await.proxy_session_id.clone(); + { + let mut promoted = promoted.write().await; + promoted.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( + Arc::clone(&input.name), + Arc::from("account-a"), + &promoted_proxy_session_id, + displaced_proxy_session_id, + 20_000, + 2_000, + )); + } + let displaced_generation = displaced.read().await.activity.origin_work_generation; + + super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await; + + let displaced_binding_mode = + displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!( + displaced_binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(displaced.read().await.activity.origin_work_generation, displaced_generation + 1); + let promoted_binding_mode = promoted.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!(promoted_binding_mode, HlsOriginAccountBindingMode::Active)); + } + + #[tokio::test] + async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_authorized_manifest_work() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(1); + session.mark_authorized_media_access(1_000); + } + let old_generation = session.read().await.activity.origin_work_generation; + + super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await; + + { + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().expect("detached binding is retained"); + assert!(matches!( + binding.binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(session.activity.origin_work_generation, old_generation + 1); + } + + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let prepared_origin = super::prepare_hls_origin_runtime( + &app_state, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_session_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::HlsOriginWorkKind::Manifest, + super::HlsOriginWorkClass::ManifestInteractive, + 4_100, + ) + .await + .expect("authorized origin work can reacquire a detached binding"); + + let binding = prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); + assert_eq!(binding.account_name.as_ref(), "account-a"); + assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + } + + #[tokio::test] + async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work() { + let input = single_hls_provider_input("soft-overlap-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; + let old_proxy_session_id = old_session.read().await.proxy_session_id.clone(); + { + let mut session = old_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + } + let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); + app_state + .active_provider + .refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60) + .await; + + let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; + let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); + let prepared_origin = super::prepare_hls_origin_runtime( + &app_state, + &new_session, + &input, + "http://account.example.com/live/account-user/account-pass/new.m3u8", + "http://account.example.com/live/account-user/account-pass/new.m3u8", + &new_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55201)), + ConnectionKind::Normal, + 0, + super::HlsOriginWorkKind::Manifest, + super::HlsOriginWorkClass::ManifestInteractive, + 21_000, + ) + .await + .expect("interactive work should use soft-active overlap before grace"); + + let binding = prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); + assert_eq!(binding.account_name, input.name); + assert!(matches!( + &binding.binding_mode, + HlsOriginAccountBindingMode::Speculative { + displaced_proxy_session_id, + .. + } if displaced_proxy_session_id == &old_proxy_session_id + )); + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::ProviderAllocation::Available(_)) + )); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + } + + #[tokio::test] + async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { + let input = single_hls_provider_input("policy-preempt-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let soft_session = create_bound_hls_test_session(&app_state, &input, "soft", input.name.as_ref(), 1_000).await; + let soft_generation = { + let mut session = soft_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(10_000); + session.reconcile_effective_origin_acquire_policy( + Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Soft, 0, 10_000)), + 10_000, + ); + session.activity.origin_work_generation + }; + let soft_binding = soft_session.read().await.origin_account_binding.clone().expect("soft binding exists"); + app_state + .active_provider + .refresh_provider_reservation(&soft_binding.account_name, &soft_binding.session_owner, 60) + .await; + + let normal_session = create_unbound_hls_test_session(&app_state, &input, "normal", 10_500).await; + let normal_proxy_session_id = normal_session.read().await.proxy_session_id.clone(); + let prepared_origin = super::prepare_hls_origin_runtime( + &app_state, + &normal_session, + &input, + "http://account.example.com/live/account-user/account-pass/normal.m3u8", + "http://account.example.com/live/account-user/account-pass/normal.m3u8", + &normal_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55231)), + ConnectionKind::Normal, + 0, + super::HlsOriginWorkKind::Manifest, + super::HlsOriginWorkClass::ManifestInteractive, + 10_500, + ) + .await + .expect("normal HLS policy should preempt active soft HLS binding"); + + let new_binding = prepared_origin + .origin_account_binding_to_store + .as_ref() + .expect("preempting session should receive active binding"); + assert_eq!(new_binding.account_name, input.name); + assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::ProviderAllocation::Available(_)) + )); + let soft_session = soft_session.read().await; + assert_eq!(soft_session.activity.origin_work_generation, soft_generation + 1); + assert!(matches!( + soft_session.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), + Some(HlsOriginAccountBindingMode::Detached { + reason: HlsOriginAccountDetachedReason::PreemptedByHigherPriority, + .. + }) + )); + drop(soft_session); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + } + + #[tokio::test] + async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal_binding() { + let input = single_hls_provider_input("policy-no-preempt-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let normal_session = create_bound_hls_test_session(&app_state, &input, "normal", input.name.as_ref(), 1_000).await; + { + let mut session = normal_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(10_000); + session.reconcile_effective_origin_acquire_policy( + Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Normal, 0, 10_000)), + 10_000, + ); + } + let normal_binding = normal_session.read().await.origin_account_binding.clone().expect("normal binding exists"); + app_state + .active_provider + .refresh_provider_reservation(&normal_binding.account_name, &normal_binding.session_owner, 60) + .await; + let soft_session = create_unbound_hls_test_session(&app_state, &input, "soft", 10_500).await; + let soft_proxy_session_id = soft_session.read().await.proxy_session_id.clone(); + + let result = super::prepare_hls_origin_policy_preempt_runtime( + &app_state, + &soft_session, + &input, + "http://account.example.com/live/account-user/account-pass/soft.m3u8", + "http://account.example.com/live/account-user/account-pass/soft.m3u8", + &soft_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55232)), + ConnectionKind::Soft, + -100, + 10_500, + ) + .await; + + assert!(result.is_err()); + assert!(matches!( + normal_session.read().await.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), + Some(HlsOriginAccountBindingMode::Active) + )); + assert!( + app_state + .active_provider + .is_provider_reserved_for_other_session(&normal_binding.account_name, Some("unrelated-session")) + .await + ); + } + + #[tokio::test] + async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fails() { + let input = single_hls_provider_input("grace-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let occupied = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55211), + false, + 0, + ConnectionKind::Normal, + Some("external-owner"), + ) + .await + .expect("test should occupy the only provider account"); + let session = create_unbound_hls_test_session(&app_state, &input, "12345", 2_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + + let prepared_origin = super::prepare_hls_origin_runtime( + &app_state, + &session, + &input, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + &proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55212)), + ConnectionKind::Normal, + 0, + super::HlsOriginWorkKind::Manifest, + super::HlsOriginWorkClass::ManifestInteractive, + 2_000, + ) + .await + .expect("interactive work can use grace when normal acquire and overlap fail"); + + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::ProviderAllocation::GracePeriod(_)) + )); + assert_eq!( + prepared_origin + .origin_account_binding_to_store + .as_ref() + .expect("grace binding should still bind the selected account") + .account_name + .as_ref(), + input.name.as_ref() + ); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle).await; + app_state.connection_manager.release_provider_handle(Some(occupied)).await; + } + + #[tokio::test] + async fn hls_provider_exhausted_without_provisioning_returns_custom_manifest() { + let input = single_hls_provider_input("provider-exhausted-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + store_test_sources_with_target( + &app_state, + input.clone(), + ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), + ); + enable_provider_exhausted_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "provider-exhausted-session", 1_000).await; + let access_lease_id = HlsAccessLeaseId("provider-exhausted-lease".to_string()); + let strip = app_state.hls_proxy.strip(); + + let response = super::hls_shared_provisioning_or_provider_exhausted_response( + &app_state, + &session, + "hls-user", + &input, + 59, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body(response).await; + let manifest = String::from_utf8(body.to_vec()).expect("manifest is utf8"); + assert!(manifest.contains("#EXTM3U")); + assert!(manifest.contains("/cvs/hls/hls-user/hls-pass/provider_connections_exhausted.ts")); + } + + #[tokio::test] + async fn hls_provider_exhausted_grace_hold_waits_for_grace_period_before_retry() { + let input = single_hls_provider_input("provider-grace-hold-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + app_state.app_config.config.store(Arc::new(Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + stream: Some(StreamConfigDto { + grace_period_millis: 20, + grace_period_timeout_secs: 10, + grace_period_hold_stream: true, + ..Default::default() + }), + ..Default::default() + })), + ..Default::default() + })); + let session = create_unbound_hls_test_session(&app_state, &input, "provider-grace-session", 1_000).await; + let access_lease_id = HlsAccessLeaseId("provider-grace-lease".to_string()); + let strip = app_state.hls_proxy.strip(); + let resolution = super::hls_provider_connections_exhausted_manifest_resolution( + &app_state, + &session, + "hls-user", + &input, + 59, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + true, + ); + + let resolution = tokio::time::timeout(Duration::from_millis(500), resolution) + .await + .expect("grace hold deadline should wake"); + + assert!(matches!(resolution, super::HlsProviderExhaustedResolution::RetryAcquire)); + } + + #[tokio::test] + async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segments() { + let input = single_hls_provider_input("shared-provisioning-timeline-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let access_lease_id = HlsAccessLeaseId("timeline-lease".to_string()); + let strip = app_state.hls_proxy.strip(); + + let response = super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); + assert!(body.contains("#EXT-X-VERSION:7\n")); + assert!(body.contains("#EXT-X-INDEPENDENT-SEGMENTS\n")); + assert!(body.contains("#EXT-X-TARGETDURATION:2\n")); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("#EXTINF:2.000,\n")); + assert!(!body.contains("#EXTINF:12.000,\n")); + assert!(body.contains("/hls/shared/live/")); + assert!(body.contains("/000000.ts?pseq=0")); + assert!(body.contains("/000001.ts?pseq=1")); + assert!(body.contains("/000002.ts?pseq=2")); + assert!(body.matches("#EXTINF:").count() <= 6); + assert!(!body.contains("/cvs/hls/")); + { + let session = session.read().await; + assert_eq!(session.proxy_next_seq, Some(3)); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(0)); + assert_eq!(session.publishable_origin_tail_proxy_seq, Some(2)); + assert_eq!(session.segments.len(), 3); + } + + let response = super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("subsequent manifest should append one segment"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); + assert!(body.contains("/000003.ts?pseq=3")); + assert!(body.matches("#EXTINF:").count() <= 6); + assert_eq!(session.read().await.proxy_next_seq, Some(4)); + } + + #[tokio::test] + async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { + let input = single_hls_provider_input("stale-provisioning-handoff-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let initial_lease_id = HlsAccessLeaseId("initial-provisioning-lease".to_string()); + let strip = app_state.hls_proxy.strip(); + + super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &initial_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render local segments"); + { + let session_guard = session.read().await; + assert!(session_guard + .segments + .values() + .any(crate::api::model::is_hls_provisioning_segment)); + assert_eq!(session_guard.segments.len(), 3); + assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); + } + + let new_lease_id = HlsAccessLeaseId("new-playback-lease".to_string()); + let previous_rendered_at = super::maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + &app_state, + &session, + &input, + 12345, + &new_lease_id, + 2_000, + ) + .await; + + assert_eq!(previous_rendered_at, None); + let session_guard = session.read().await; + assert_eq!(session_guard.segments.len(), 3); + assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); + } + + #[tokio::test] + async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segments() { + let input = single_hls_provider_input("shared-provisioning-handoff-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "12345", 1_000).await; + let access_lease_id = HlsAccessLeaseId("handoff-lease".to_string()); + let strip = app_state.hls_proxy.strip(); + super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render"); + + let manifest = match parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4025\n#EXTINF:4.0,\n4025.ts\n#EXTINF:4.0,\n4026.ts\n#EXTINF:4.0,\n4027.ts\n", + "http://origin.example/live/stream.m3u8", + ) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + }; + let rendered = { + let mut session_guard = session.write().await; + session_guard.mark_pending_handoff_discontinuity(0); + drop(session_guard); + assert!( + super::ensure_shared_hls_provisioning_handoff_gap(&app_state, &session, 2_000).await, + "handoff should append one gap segment" + ); + let mut session_guard = session.write().await; + session_guard.apply_origin_manifest(&manifest).expect("origin manifest should map"); + for proxy_seq in 4..=6 { + session_guard.segments.get_mut(&proxy_seq).expect("origin segment").status = + SegmentCacheStatus::Ready { content_length: 1024, ready_at_ms: 2_000 }; + } + session_guard.render_and_store_manifest(2_000).expect("handoff manifest should render") + }; + + assert!(rendered.body.contains("#EXT-X-MEDIA-SEQUENCE:1\n")); + assert!(rendered.body.contains("#EXT-X-TARGETDURATION:4\n")); + assert!(rendered.body.contains("/000002.ts?pseq=2")); + assert!(rendered.body.contains("/000004.ts")); + assert!(rendered.body.contains("/000005.ts")); + assert!(rendered.body.contains("/000006.ts")); + assert!(!rendered.body.contains("/004025.ts")); + let provisioning_tail = rendered.body.find("/000002.ts?pseq=2").expect("provisioning tail is rendered"); + let gap_tag = rendered.body.find("#EXT-X-GAP\n").expect("handoff gap tag is rendered"); + let gap_uri = rendered.body.find("/000003.ts?pseq=3").expect("handoff gap uri is rendered"); + let discontinuity = rendered + .body + .find("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,\n/hls/shared/live/") + .expect("origin handoff discontinuity is rendered"); + let first_origin = rendered.body.find("/000004.ts").expect("first origin segment is rendered"); + assert!(provisioning_tail < gap_tag); + assert!(gap_tag < gap_uri); + assert!(gap_uri < discontinuity); + assert!(discontinuity < first_origin); + } + + #[tokio::test] + async fn hls_origin_runtime_background_skips_soft_overlap_and_grace() { + let input = single_hls_provider_input("background-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; + { + let mut session = old_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + } + let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); + app_state + .active_provider + .refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60) + .await; + let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; + let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); + + let result = super::prepare_hls_origin_runtime( + &app_state, + &new_session, + &input, + "http://account.example.com/live/account-user/account-pass/new.m3u8", + "http://account.example.com/live/account-user/account-pass/new.m3u8", + &new_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55221)), + ConnectionKind::Normal, + 0, + super::HlsOriginWorkKind::Segment, + super::HlsOriginWorkClass::Background, + 12_000, + ) + .await; + + assert_eq!( + result.err(), + Some(super::HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted + }) + ); + let old_session = old_session.read().await; + assert!(matches!( + old_session.origin_account_binding.as_ref().expect("old binding remains").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert!(new_session.read().await.origin_account_binding.is_none()); + } + + #[test] + fn hls_detached_origin_binding_reclaimed_by_owner_maps_to_preempted_no_account_reason() { + let proxy_session_id = ProxySessionId("preempted-session".to_string()); + let mut binding = HlsOriginAccountBinding::new( + Arc::from("input-a"), + Arc::from("account-a"), + &proxy_session_id, + 1_000, + ); + binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, 2_000); + + assert_eq!( + super::hls_no_account_reason_for_binding(Some(&binding)), + super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted + ); + } + + #[test] + fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_reason() { + let proxy_session_id = ProxySessionId("soft-window-session".to_string()); + let mut binding = HlsOriginAccountBinding::new( + Arc::from("input-a"), + Arc::from("account-a"), + &proxy_session_id, + 1_000, + ); + binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, 2_000); + + assert_eq!( + super::hls_no_account_reason_for_binding(Some(&binding)), + super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted + ); + } + + #[tokio::test] + async fn hls_account_binding_without_media_activity_is_not_detached() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(12); + session.mark_authorized_manifest_access(1_000); + } + let old_generation = session.read().await.activity.origin_work_generation; + + super::detach_unprotected_hls_origin_account_bindings(&app_state, 60_000).await; + + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().expect("binding remains"); + assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert_eq!(session.activity.origin_work_generation, old_generation); + assert_eq!(session.activity.last_authorized_media_at_ms, None); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); + } + + #[tokio::test] + async fn hls_hard_manifest_failure_forces_next_fresh_commit() { + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: 5_000, + render_gap_segments: 0, + rendered_at_ms: 1_000, + segment_proxy_seqs: vec![1], + }); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, 2_000).await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure + } + ); + } + + #[tokio::test] + async fn hls_normal_expired_session_allows_committed_manifest_while_manifest_valid() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_add(10_000), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(1_000), + segment_proxy_seqs: vec![1], + }); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::CommittedManifestAllowed + ); + } + + #[tokio::test] + async fn hls_normal_expired_session_requires_fresh_commit_after_manifest_validity() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_sub(1), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(10_000), + segment_proxy_seqs: vec![1], + }); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation + } + ); + } + + #[tokio::test] + async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + { + let mut session = session.write().await; + session + .origin_account_binding + .as_mut() + .expect("binding exists") + .detach(HlsOriginAccountDetachedReason::Cleanup, 2_000); + session.segments.insert( + 123, + SegmentEntry { + origin_key: OriginSegmentKey { origin_epoch: 0, origin_seq: 123 }, + proxy_seq: 123, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(proxy_session_id, 123, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + origin_fetch_ref: Some(OriginSegmentFetchRef { + resolved_origin_url: "http://origin.example.com/123.ts".to_string(), + byte_range: None, + valid_until_ms: None, + }), + status: SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }, + ); + } + + let segment_file = HlsSegmentFile { proxy_seq: 123, extension: "ts".to_string() }; + assert!(!super::hls_segment_request_requires_origin_work(&session, &segment_file).await); + assert!(super::hls_origin_binding_needs_reacquire(&session).await); + } + + async fn activate_test_hls_access_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &str, + now_ms: u64, + ttl_ms: u64, + ) { + let lease_id = HlsAccessLeaseId(lease_id.to_string()); + let valid_window_ms = ttl_ms.saturating_mul(10).max(ttl_ms); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + valid_window_ms, + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: ttl_ms, valid_window_ms }, + ) + .await + .is_activated()); + } + + async fn register_test_hls_stream_for_lease_release( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + provider: &str, + ) { + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.max_connections = 1; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "hls-session-token", + virtual_id: 12345, + provider, + stream_url: "http://origin.example.com/live/user/pass/12345.m3u8", + addr: &test_addr(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let mut stream_channel = + super::fallback_hls_cache_stream_channel(0, 12345, &session.read().await.origin_source, proxy_session_id); + stream_channel.shared = true; + stream_channel.shared_stream_id = Some(super::hls_cache_shared_stream_id(proxy_session_id)); + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &test_fingerprint(), + provider: Arc::from(provider), + stream_channel: &stream_channel, + user_agent: std::borrow::Cow::Borrowed("test"), + session_token: Some("hls-session-token"), + }) + .await; + } + + fn test_segment_entry( + proxy_session_id: &ProxySessionId, + proxy_seq: u64, + status: SegmentCacheStatus, + ) -> SegmentEntry { + SegmentEntry { + origin_key: OriginSegmentKey { origin_epoch: 0, origin_seq: proxy_seq }, + proxy_seq, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(proxy_session_id.clone(), proxy_seq, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + origin_fetch_ref: Some(OriginSegmentFetchRef { + resolved_origin_url: format!("http://origin.example.com/{proxy_seq}.ts"), + byte_range: None, + valid_until_ms: None, + }), + status, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + } + } + + #[tokio::test] + async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); + let account_name = Arc::from("account-a"); + app_state.active_provider.refresh_provider_reservation(&account_name, &binding.session_owner, 60).await; + assert!( + app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")).await + ); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "detach-lease", 1_000, 1_000).await; + register_test_hls_stream_for_lease_release(&app_state, &session, &proxy_session_id, input.name.as_ref()).await; + assert_eq!(app_state.active_users.active_streams().await.len(), 1); + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_000, + ) + .await; + { + let mut session = session.write().await; + assert_eq!(session.activity.active_access_lease_count, 1); + session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); + session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); + session.segments.insert( + 2, + test_segment_entry( + &proxy_session_id, + 2, + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, + ), + ); + let map_id = ProxyMapId(1); + let mut map = MapEntry::new( + &proxy_session_id, + map_id, + OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "http://origin.example.com/init.mp4".to_string(), + byte_range: None, + }, + "mp4".to_string(), + ); + map.status = MapCacheStatus::Queued { queued_at_ms: 1_100 }; + session.maps.insert(map_id, map); + } + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 3_000, + ) + .await; + + { + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + let binding = session.origin_account_binding.as_ref().expect("binding is retained"); + assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert_eq!(session.activity.origin_work_generation, old_generation); + assert!(!session.segment_prefetch_queue.is_empty()); + assert!(matches!( + session.segments.get(&1).expect("queued segment remains").status, + SegmentCacheStatus::Queued { .. } + )); + assert!(matches!( + session.segments.get(&2).expect("ready segment remains").status, + SegmentCacheStatus::Ready { .. } + )); + assert!(matches!( + session.maps.get(&ProxyMapId(1)).expect("map remains").status, + MapCacheStatus::Queued { .. } + )); + } + assert!( + app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")).await + ); + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + #[tokio::test] + async fn hls_access_lease_sync_keeps_binding_and_queue_when_active_count_remains_positive() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "still-active-lease", 1_000, 10_000).await; + { + let mut session = session.write().await; + session.activity.active_access_lease_count = 1; + session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); + session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); + } + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_500, + ) + .await; + + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 1); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); + assert!(!session.segment_prefetch_queue.is_empty()); + assert!(matches!( + session.segments.get(&1).expect("segment remains queued").status, + SegmentCacheStatus::Queued { .. } + )); + } + + #[tokio::test] + async fn hls_access_lease_sync_zero_to_zero_is_idempotent() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 3_000, + ) + .await; + + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); + } + + #[tokio::test] + async fn hls_access_lease_gc_prepass_releases_user_without_detaching_origin_binding() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "gc-expired-lease", 1_000, 1_000).await; + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_000, + ) + .await; + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls_proxy + .sync_all_session_access_leases_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + 3_000, + ) + .await; + let _ = app_state.hls_proxy.run_garbage_collection_once(3_000).await.expect("gc should run"); + + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("detach keeps shared hls session"); + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); + } + + fn test_addr() -> SocketAddr { "127.0.0.1:55123".parse().unwrap_or_else(|_| unreachable!()) } + + fn test_addr_with_port(port: u16) -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], port)) } + + fn test_fingerprint() -> Fingerprint { Fingerprint::new("test".to_string(), "127.0.0.1".to_string(), test_addr()) } + + fn test_fingerprint_with_addr(addr: SocketAddr) -> Fingerprint { + Fingerprint::new(format!("test-{}", addr.port()), "127.0.0.1".to_string(), addr) + } + + async fn create_active_hls_user_session(app_state: &Arc) { + create_active_hls_user_session_with( + app_state, + "hls-session-token", + "origin-provider", + "http://origin.example.com/live/12345.m3u8", + test_addr(), + ) + .await; + } + + async fn create_active_hls_user_session_with( + app_state: &Arc, + session_token: &str, + provider: &str, + stream_url: &str, + addr: SocketAddr, + ) { + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.max_connections = 1; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token, + virtual_id: 12345, + provider, + stream_url, + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + } + + fn test_hls_access_context( + proxy_session_id: ProxySessionId, + access_lease_id: HlsAccessLeaseId, + ) -> HlsAccessContext { + test_hls_access_context_with(proxy_session_id, access_lease_id, "hls-session-token", test_fingerprint().key) + } + + fn test_hls_access_context_with( + proxy_session_id: ProxySessionId, + access_lease_id: HlsAccessLeaseId, + user_session_token: &str, + client_fingerprint: String, + ) -> HlsAccessContext { + HlsAccessContext { + username: "hls-user".to_string(), + user_session_token: user_session_token.to_string(), + proxy_session_id, + input_id: 1, + stream_ref: "12345".to_string(), + virtual_id: 12345, + lease_id: access_lease_id, + family_key: HlsPlaybackFamilyKey::new("hls-user", client_fingerprint), + } + } + + async fn prepare_pending_test_hls_access_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + ) { + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + access_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + super::current_time_millis(), + super::hls_access_lease_ttl_ms(app_state), + )) + .await; + } + + #[tokio::test] + async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input_name = Arc::::from("test-input"); + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(body.contains(&format!("/hls/shared/live/{}/{}/000000.ts", proxy_session_id.0, access_lease_id.0))); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + let session = app_state + .hls_proxy + .sessions() + .get_by_key(&session_key) + .await + .expect("cold start should create shared hls session"); + { + let session = session.read().await; + assert!(session.last_rendered_manifest.is_some()); + let binding = session.origin_account_binding.as_ref().expect("plain http input still has account binding"); + assert_eq!(binding.input_name.as_ref(), "test-input"); + assert_eq!(binding.account_name.as_ref(), "test-input"); + } + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + #[tokio::test] + async fn canonical_recovery_from_provisioning_marks_normal_handoff_boundary() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::current_time_millis()); + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("canonical hls cache should recover from provisioning"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,")); + assert!(!app_state.hls_provisioning.has_consumer(&input.name, 12345, super::current_time_millis())); + } + + #[tokio::test] + async fn canonical_recovery_from_provisioning_marks_transient_handoff_boundary() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.key\"\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + app_state.hls_provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::current_time_millis()); + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Activated, + super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("canonical hls cache should recover from provisioning"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.0,")); + assert!(!app_state.hls_provisioning.has_consumer(&input.name, 12345, super::current_time_millis())); + } + + #[tokio::test] + async fn hls_cache_manifest_cold_start_supports_m3u_hls_origin_source() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:77\n#EXTINF:4.0,\nseg-77.ts\n#EXTINF:4.0,\nseg-78.ts\n#EXTINF:4.0,\nseg-79.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("m3u-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("hls cache should handle m3u hls media playlists"); + + assert_eq!(response.status(), StatusCode::OK); + let session = app_state + .hls_proxy + .sessions() + .get_by_key(&session_key) + .await + .expect("m3u hls should create shared session"); + let session = session.read().await; + assert_eq!(session.origin_source.source_kind, crate::api::model::HlsOriginSourceKind::M3uMediaPlaylist); + let binding = session.origin_account_binding.as_ref().expect("m3u hls input still has account binding"); + assert_eq!(binding.input_name.as_ref(), "m3u-input"); + assert_eq!(binding.account_name.as_ref(), "m3u-input"); + } + + #[tokio::test] + async fn hls_cache_manifest_cold_start_unreachable_origin_redirects_to_channel_unavailable_custom_response() { + let input_name = Arc::::from("test-input"); + let origin = spawn_test_status_origin(StatusCode::NOT_FOUND, b"missing").await; + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + enable_channel_unavailable_custom_response(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: request_url.as_str(), + session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url.as_str()), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + assert!( + response.headers().get(header::RETRY_AFTER).is_none(), + "custom response must not expose retry-after" + ); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + + let mut marked = false; + for _ in 0..50 { + let snapshot = app_state + .hls_proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) + .await; + if snapshot.is_some_and(|lease| { + matches!( + lease.response_flag, + Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { + reason: HlsAccessLeaseChannelUnavailableReason::ManifestCommitFailed { + reason: HlsFreshManifestRequiredReason::ColdStart + }, + .. + }) + ) + }) { + marked = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(marked, "cold-start manifest failure should mark the access lease channel unavailable"); + } + + #[tokio::test] + async fn hls_cache_manifest_cold_start_client_abort_does_not_leave_refresh_in_flight() { + let origin = spawn_test_transient_origin_with_delayed_response( + "200 OK", + &[("Content-Type", "application/vnd.apple.mpegurl")], + "#EXTM3U\n#EXT-X-VERSION:3\n", + Duration::from_millis(100), + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let hls_dto = HlsCacheConfigDto { origin_manifest_timeout_ms: 1_000, ..Default::default() }; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config)), + vec![Arc::new(input.clone())], + ); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let app_state_for_request = Arc::clone(&app_state); + let request_handle = tokio::spawn(async move { + super::try_hls_cache_canonical_manifest_response( + &app_state_for_request, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + }); + + let session = wait_for_hls_test_session(&app_state, &session_key).await; + wait_for_hls_refresh_in_flight(&session).await; + request_handle.abort(); + let _ = request_handle.await; + + for _ in 0..200 { + if !session.read().await.origin_refresh.in_flight { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let session = session.read().await; + assert!(!session.origin_refresh.in_flight); + assert!(session.origin_refresh.last_fetch_finished_at_ms.is_some()); + } + + #[tokio::test] + async fn hls_cache_canonical_prepare_service_unavailable_sets_retry_after() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let input = ConfigInput { + id: 1, + name: Arc::from("alias-input"), + aliases: Some(vec![crate::model::ConfigInputAlias { + id: 2, + name: Arc::from("alias-account"), + url: "http://alias.example.com".to_string(), + username: Some("alias-user".to_string()), + password: Some("alias-pass".to_string()), + priority: 0, + max_connections: 1, + exp_date: None, + enabled: true, + }]), + ..ConfigInput::default() + }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("canonical hls cache response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); + } + + #[tokio::test] + async fn hls_origin_account_rebind_failure_sets_backoff_without_changing_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let input = ConfigInput { id: 1, name: Arc::from("stale-input"), ..ConfigInput::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + let (session, _) = app_state + .hls_proxy + .get_or_create_session_with_source_and_outcome( + session_key.clone(), + origin_source.clone(), + &app_state.get_encrypt_secret(), + 1_000, + ) + .await; + { + let mut session_guard = session.write().await; + session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("removed-account"), + &proxy_session_id, + 1_000, + )); + } + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: request_url, + session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("canonical hls cache response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); + let session_guard = session.read().await; + assert_eq!(session_guard.key, session_key); + assert_eq!(session_guard.proxy_session_id, proxy_session_id); + let binding = session_guard.origin_account_binding.as_ref().expect("stale binding remains"); + assert_eq!(binding.account_name.as_ref(), "removed-account"); + assert_eq!(binding.generation, 0); + assert_eq!(session_guard.origin_account_rebind.consecutive_rebind_failures, 1); + assert!(session_guard.origin_account_rebind.next_rebind_allowed_at_ms.is_some()); + } + + #[tokio::test] + async fn hls_cache_entry_returns_temporary_redirect_without_origin_refresh_or_session_creation() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + + let response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + Some("/iptv"), + ) + .await + .expect("hls cache entry should redirect"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + assert_eq!(response.headers().get(header::CACHE_CONTROL).expect("cache control"), "no-store"); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + assert!(location.starts_with("/iptv/hls/shared/live/")); + assert!(location.ends_with("/manifest.m3u8")); + let access_lease_id = + location.trim_end_matches("/manifest.m3u8").rsplit('/').next().expect("access lease id in redirect"); + assert_eq!(access_lease_id.len(), 22); + assert!(app_state.hls_proxy.sessions().get_by_key(&session_key).await.is_none()); + assert_eq!(app_state.hls_proxy.metrics().snapshot().refresh_started, 0); + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + #[tokio::test] + async fn hls_cache_entry_denies_access_lease_when_grace_has_no_origin_slot() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + + let response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::GracePeriod, + None, + Some("/iptv"), + ) + .await + .expect("hls cache entry should redirect to canonical denied lease"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(location).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(location).to_string()); + let now_ms = super::current_time_millis(); + let snapshot = app_state + .hls_proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) + .await + .expect("denied lease should stay available for response rendering"); + assert_eq!(snapshot.state, HlsAccessLeaseState::Denied); + assert!(app_state + .hls_proxy + .access_lease_session_snapshot(&proxy_session_id, now_ms) + .await + .effective_origin_policy + .is_none()); + + let err = super::validate_hls_proxy_access_context( + &app_state, + &test_fingerprint(), + &proxy_session_id, + &access_lease_id.0, + now_ms, + crate::api::model::HlsAccessAdmissionMode::ManifestPrepare, + ) + .await + .expect_err("denied lease must surface as admission denied"); + assert_eq!(err, HlsAccessLeaseValidationError::AdmissionDenied); + } + + #[tokio::test] + async fn hls_cache_entry_redirect_uses_cache_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + + let response = super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + "http://origin.example.com/live/user/pass/12345.m3u8", + None, + 12345, + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + assert!(location.starts_with("/hls/shared/live/")); + assert!(location.ends_with("/manifest.m3u8")); + assert_eq!(app_state.hls_proxy.access_leases().read().await.len(), 1); + } + + #[tokio::test] + async fn hls_cache_entry_uses_legacy_path_when_target_hls_share_disabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(false); + let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + + let response = super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + "http://origin.example.com/live/user/pass/12345.m3u8", + None, + 12345, + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + ) + .await + .into_response(); + + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(!location.contains("/hls/shared/live/")); + assert!(app_state.hls_proxy.access_leases().read().await.is_empty()); + assert_eq!(app_state.hls_proxy.metrics().snapshot().refresh_started, 0); + } + + #[tokio::test] + async fn legacy_hls_token_route_redirects_to_channel_unavailable_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + app_state.app_config.config.store(Arc::new(Config { + custom_stream_response_enabled: true, + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + })), + ..Default::default() + })); + configure_default_test_server(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = test_hls_input(); + let target = test_hls_share_target(true); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let encrypt_secret = app_state.get_encrypt_secret(); + let legacy_manifest = rewrite_hls( + &user, + &RewriteHlsProps { + secret: &encrypt_secret, + base_url: "", + content: "#EXTM3U\n#EXTINF:4.0,\nseg.ts\n", + hls_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), + target_id: target.id, + virtual_id: 12345, + input_id: input.id, + user_token: Some("legacy-session-token"), + }, + ); + let token = legacy_manifest + .lines() + .find_map(|line| line.rsplit_once('/').map(|(_, token)| token.trim().to_string())) + .expect("legacy hls segment token should be rendered"); + + let response = super::hls_api_stream_resolved( + test_fingerprint(), + HeaderMap::new(), + Arc::clone(&app_state), + Arc::clone(&user), + Arc::new(target), + input.id, + 12345, + token, + ) + .await; + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + assert!(app_state.hls_proxy.access_leases().read().await.is_empty()); + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + #[tokio::test] + async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = test_hls_input(); + let target = test_hls_share_target(true); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + + let response = super::hls_api_stream_resolved( + test_fingerprint(), + HeaderMap::new(), + Arc::clone(&app_state), + user, + Arc::new(target), + input.id, + 12345, + "not-a-valid-legacy-token.ts".to_string(), + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(app_state.hls_proxy.access_leases().read().await.is_empty()); + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + #[tokio::test] + async fn hls_cache_canonical_manifest_rejects_when_target_hls_share_disabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let input = test_hls_input(); + let target = test_hls_share_target(false); + store_test_sources_with_target(&app_state, input, target); + let access_context = HlsAccessContext { + username: "hls-user".to_string(), + user_session_token: "hls-session-token".to_string(), + proxy_session_id: ProxySessionId("proxy-session".to_string()), + input_id: 1, + stream_ref: "12345".to_string(), + virtual_id: 12345, + lease_id: HlsAccessLeaseId("access-lease".to_string()), + family_key: HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + }; + + let Err(err) = + super::resolve_hls_playback_manifest_request_context(&app_state, &access_context, &HeaderMap::new()).await + else { + panic!("disabled target hls sharing should reject canonical cache path"); + }; + + assert_eq!(err, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn hls_cache_entry_leases_update_effective_origin_acquire_policy_for_shared_session() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut soft_user = ProxyUserCredentials::default(); + soft_user.username = "soft-user".to_string(); + soft_user.soft_priority = 20; + let mut normal_user = ProxyUserCredentials::default(); + normal_user.username = "normal-user".to_string(); + normal_user.priority = -5; + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + + let soft_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &soft_user, + origin_source.clone(), + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Soft), + None, + ) + .await + .expect("soft hls cache entry should redirect"); + assert_eq!(soft_response.status(), StatusCode::TEMPORARY_REDIRECT); + let soft_snapshot = + app_state.hls_proxy.access_lease_session_snapshot(&proxy_session_id, super::current_time_millis()).await; + let soft_policy = soft_snapshot.effective_origin_policy.expect("soft policy"); + assert_eq!(soft_policy.connection_kind, ConnectionKind::Soft); + assert_eq!(soft_policy.priority, soft_user.soft_priority); + + // Different user/family, same shared HLS session. Normal media admission must upgrade + // the future origin-account acquire policy without changing the shared session identity. + let normal_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &normal_user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("normal hls cache entry should redirect"); + assert_eq!(normal_response.status(), StatusCode::TEMPORARY_REDIRECT); + let normal_snapshot = + app_state.hls_proxy.access_lease_session_snapshot(&proxy_session_id, super::current_time_millis()).await; + let normal_policy = normal_snapshot.effective_origin_policy.expect("normal policy"); + assert_eq!(normal_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(normal_policy.priority, normal_user.priority); + + let (session, _) = app_state + .hls_proxy + .get_or_create_session_with_source_and_outcome( + session_key, + super::build_hls_origin_source(&input, "12345"), + &app_state.get_encrypt_secret(), + super::current_time_millis(), + ) + .await; + app_state + .hls_proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + super::current_time_millis(), + ) + .await; + let session_policy = session.read().await.effective_origin_acquire_policy_or_default(); + assert_eq!(session_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(session_policy.priority, normal_user.priority); + } + + #[tokio::test] + async fn hls_entry_origin_reservation_requires_real_provider_handle() { + let app_state = test_app_state(); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = single_hls_provider_input("missing-provider"); + + let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &input, + 12345, + "http://origin.example.com/live/source-user/source-pass/12345.m3u8", + "hls-session-token", + "hls-cache:test-session", + super::hls_origin_account_reservation_ttl_secs_fallback(), + UserConnectionPermission::Allowed, + ConnectionKind::Normal, + false, + ) + .await; + + assert!(reservation.is_none(), "provisioning redirect must not use an exhausted/counter-only check"); + } + + #[tokio::test] + async fn hls_entry_origin_reservation_sets_owner_reservation_before_redirect() { + let input = single_hls_provider_input("available-provider"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let session_owner = "hls-cache:test-session"; + + let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &input, + 12345, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "hls-session-token", + session_owner, + super::hls_origin_account_reservation_ttl_secs_fallback(), + UserConnectionPermission::Allowed, + ConnectionKind::Normal, + false, + ) + .await + .expect("provider reservation should be acquired before provisioning redirect"); + + assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); + assert!(reservation.selected_provider_config.is_some()); + assert!(app_state.active_users.active_streams().await.is_empty()); + app_state.connection_manager.release_provider_handle(reservation.provider_handle).await; + + assert!( + app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55251), + false, + 0, + ConnectionKind::Normal, + Some("other-owner"), + ) + .await + .is_none(), + "reserved provider must stay blocked for other HLS sessions" + ); + + let same_owner_handle = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55252), + false, + 0, + ConnectionKind::Normal, + Some(session_owner), + ) + .await; + assert!(same_owner_handle.is_some(), "reserved provider must be reusable by the same HLS session owner"); + app_state.connection_manager.release_provider_handle(same_owner_handle).await; + } + + #[tokio::test] + async fn hls_cache_entry_creates_new_lease_for_same_user_session_and_proxy_session() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + + let first_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("first hls cache entry should redirect"); + let first_location = first_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("first location header"); + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(first_location).to_string()); + let first_access_lease_id = + HlsAccessLeaseId(access_lease_id_from_redirect_location(first_location).to_string()); + let first_session_token = + access_lease_session_token(&app_state, &proxy_session_id, &first_access_lease_id).await; + + let second_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("second hls cache entry should redirect"); + let second_location = second_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("second location header"); + assert_ne!( + access_lease_id_from_redirect_location(first_location), + access_lease_id_from_redirect_location(second_location) + ); + let second_access_lease_id = + HlsAccessLeaseId(access_lease_id_from_redirect_location(second_location).to_string()); + let second_session_token = + access_lease_session_token(&app_state, &proxy_session_id, &second_access_lease_id).await; + assert_ne!(first_session_token, second_session_token); + } + + #[tokio::test] + async fn hls_cache_entry_creates_new_lease_after_manifest_touch() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + + let first_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("first hls cache entry should redirect"); + let first_location = first_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("first location header"); + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(first_location).to_string()); + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(first_location).to_string()); + let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; + let now_ms = super::current_time_millis(); + + assert!(matches!( + app_state + .hls_proxy + .touch_manifest_access_lease( + &first_lease_id, + &proxy_session_id, + now_ms, + None, + Some(super::HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(super::hls_pending_bootstrap_window_ms()), + }), + super::hls_access_lease_ttl_ms(&app_state), + ) + .await, + crate::api::model::HlsAccessLeaseTouch::Touched { .. } + )); + + let second_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("second hls cache entry should redirect"); + let second_location = second_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("second location header"); + + assert_ne!( + access_lease_id_from_redirect_location(first_location), + access_lease_id_from_redirect_location(second_location) + ); + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(second_location).to_string()); + let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; + assert_ne!(first_session_token, second_session_token); + } + + #[tokio::test] + async fn hls_cache_entry_does_not_reuse_activated_access_lease() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + + let first_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("first hls cache entry should redirect"); + let first_location = first_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("first location header"); + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(first_location).to_string()); + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(first_location).to_string()); + let now_ms = super::current_time_millis(); + assert!(app_state + .hls_proxy + .activate_access_lease( + &first_lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::hls_access_lease_ttl_ms(&app_state), + }, + ) + .await + .is_activated()); + + let second_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("second hls cache entry should redirect"); + let second_location = second_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("second location header"); + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(second_location).to_string()); + + assert_ne!(first_lease_id, second_lease_id); + let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; + let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; + assert_ne!(first_session_token, second_session_token); + assert!( + app_state + .hls_proxy + .touch_access_lease( + &first_lease_id, + super::current_time_millis(), + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::hls_access_lease_ttl_ms(&app_state), + }, + ) + .await + ); + } + + #[tokio::test] + async fn hls_cache_entry_ignores_existing_pending_lease_for_new_playback() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + let old_lease_id = HlsAccessLeaseId("old-pending-lease".to_string()); + let old_session_token = "old-hls-session-token"; + let old_issued_at_ms = super::current_time_millis().saturating_sub(6_000); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + old_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + old_session_token.to_string(), + 1, + "12345".to_string(), + 12345, + old_issued_at_ms, + super::hls_access_lease_ttl_ms(&app_state), + )) + .await; + + let response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("hls cache entry should redirect"); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + let new_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(location).to_string()); + let new_session_token = access_lease_session_token(&app_state, &proxy_session_id, &new_lease_id).await; + + assert_ne!(old_lease_id, new_lease_id); + assert_ne!(old_session_token, new_session_token); + } + + #[tokio::test] + async fn hls_cache_parallel_real_playbacks_same_virtual_id_register_distinct_streams() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::build_hls_origin_source(&input, "12345"); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + + let first_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("first hls cache entry should redirect"); + let first_location = first_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("first location header"); + let first_lease_id = access_lease_id_from_redirect_location(first_location); + assert_eq!(proxy_session_id_from_redirect_location(first_location), proxy_session_id); + let proxy_session = ProxySessionId(proxy_session_id.clone()); + let first_session_token = + access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(first_lease_id.to_string())).await; + let first_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{first_lease_id}/000123.ts"); + assert_eq!(get_status(Arc::clone(&app_state), &first_segment_uri).await, StatusCode::OK); + + let second_response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("second hls cache entry should redirect"); + let second_location = second_response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .expect("second location header"); + let second_lease_id = access_lease_id_from_redirect_location(second_location); + assert_ne!(first_lease_id, second_lease_id); + assert_eq!(proxy_session_id_from_redirect_location(second_location), proxy_session_id); + let second_session_token = + access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(second_lease_id.to_string())) + .await; + assert_ne!(first_session_token, second_session_token); + let second_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{second_lease_id}/000123.ts"); + assert_eq!(get_status(Arc::clone(&app_state), &second_segment_uri).await, StatusCode::OK); + + let streams = app_state.active_users.active_streams().await; + assert_eq!(streams.len(), 2); + let first_stream = streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(first_session_token.as_str())) + .expect("first stream should be registered"); + let second_stream = streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(second_session_token.as_str())) + .expect("second stream should be registered"); + let shared_stream_id = super::hls_cache_shared_stream_id(&proxy_session); + assert_ne!(first_stream.session_token, second_stream.session_token); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); + } + + #[tokio::test] + async fn hls_cache_entry_redirect_for_xtream_uses_stream_ref_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { + id: 7, + name: Arc::from("xtream-input"), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let origin_source = super::build_hls_origin_source(&input, "80510"); + let expected_proxy_session_id = + build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + + let response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 80510, + None, + "http://origin.example.com/live/user/pass/80510.m3u8", + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await + .expect("xtream hls cache entry should redirect"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + assert!(location.starts_with(&format!("/hls/shared/live/{}/", expected_proxy_session_id.0))); + assert!(location.ends_with("/manifest.m3u8")); + assert!(app_state.hls_proxy.sessions().get_by_key(&HlsSessionKey::new(7, "80510")).await.is_none()); + assert_eq!(app_state.hls_proxy.metrics().snapshot().refresh_started, 0); + } + + #[tokio::test] + async fn hls_cache_entry_redirect_for_m3u_uses_stream_ref_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = + ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; + let origin_source = super::build_hls_origin_source(&input, "70001"); + let expected_proxy_session_id = + build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + + let response = super::try_hls_cache_entry_redirect( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 70001, + None, + "http://media.example.com/channel/playlist.m3u8", + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + Some("/iptv"), + ) + .await + .expect("m3u hls cache entry should redirect"); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = + response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).expect("location header"); + assert!(location.starts_with(&format!("/iptv/hls/shared/live/{}/", expected_proxy_session_id.0))); + assert!(location.ends_with("/manifest.m3u8")); + assert!(app_state.hls_proxy.sessions().get_by_key(&HlsSessionKey::new(9, "70001")).await.is_none()); + assert_eq!(app_state.hls_proxy.metrics().snapshot().refresh_started, 0); + } + + #[tokio::test] + async fn hls_proxy_manifest_invalid_token_starts_no_origin_work() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + + let response = get_response( + Arc::clone(&app_state), + "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/not-a-valid-token/manifest.m3u8", + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(app_state.hls_proxy.metrics().snapshot().refresh_started, 0); + assert!(app_state.hls_proxy.sessions().is_empty().await); + } + + #[tokio::test] + async fn hls_cache_manifest_response_applies_current_users_server_path_without_mutating_session_body() { + let input_name = Arc::::from("test-input"); + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: "http://origin.example.com".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = u64::MAX; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-MAP:URI=\"/hls/shared/live/{proxy_session_id}/{}/map/000000.mp4\"\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_id}/{}/000123.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 123, + last_proxy_seq: 123, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + segment_proxy_seqs: vec![123], + }); + proxy_session_id + }; + let proxy_session_id = ProxySessionId(proxy_session_id); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + + let response = super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::HlsCacheManifestOrigin { + raw_request_url: "http://origin.example.com/live/user/pass/12345.m3u8", + session_entry_url: super::HlsOriginEntryUrl::direct_http( + "http://origin.example.com/live/user/pass/12345.m3u8", + ), + input: &input, + origin_source: super::build_hls_origin_source(&input, "12345"), + }, + HeaderMap::new(), + Some("/iptv"), + "/live/hls-user/hls-pass/12345.m3u8", + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + + assert!(body.contains(&format!("/iptv/hls/shared/live/{}/", proxy_session_id.0))); + assert!(body.contains("/map/000000.mp4")); + assert!(body.contains("/000123.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + let stored_body = session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body.clone(); + assert!(stored_body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert!(stored_body.contains(&format!("/hls/shared/live/{}/", proxy_session_id.0))); + assert!(!stored_body.contains("/iptv/hls/shared/live/")); + assert!(!stored_body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + } + + fn transient_manifest_body(proxy_session_id: &str) -> String { + transient_manifest_body_from_sequence(proxy_session_id, 100, 6) + } + + fn transient_manifest_body_from_sequence(proxy_session_id: &str, first_sequence: u64, count: usize) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); + for index in 0..count { + let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); + body.push_str("#EXTINF:10.0,\n"); + let _ = writeln!( + body, + "/hls/shared/live/{proxy_session_id}/{}/r/seg{sequence}.ts", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + } + body + } + + fn media_uri_count(body: &str) -> usize { + body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).count() + } + + fn normal_manifest_body(proxy_session_id: &str) -> String { + normal_manifest_body_from_sequence(proxy_session_id, 0, 6) + } + + fn normal_manifest_body_from_sequence(proxy_session_id: &str, first_sequence: u64, count: usize) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); + for index in 0..count { + let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); + body.push_str("#EXTINF:10.0,\n"); + let _ = writeln!( + body, + "/hls/shared/live/{proxy_session_id}/{}/{sequence:06}.ts", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + } + body + } + + fn store_normal_manifest_body(session: &mut HlsSession, body: String, rendered_at_ms: u64) { + store_normal_manifest_body_range(session, body, 0, 6, rendered_at_ms); + } + + fn store_normal_manifest_body_range( + session: &mut HlsSession, + body: String, + first_proxy_seq: u64, + count: usize, + rendered_at_ms: u64, + ) { + let last_proxy_seq = + first_proxy_seq.saturating_add(u64::try_from(count.saturating_sub(1)).expect("test count fits u64")); + session.last_rendered_manifest = Some(RenderedManifest { + body, + first_proxy_seq, + last_proxy_seq, + playlist_duration_ms: 60_000, + valid_until_ms: rendered_at_ms.saturating_add(60_000), + render_gap_segments: 0, + rendered_at_ms, + segment_proxy_seqs: (first_proxy_seq..=last_proxy_seq).collect(), + }); + } + + #[tokio::test] + async fn hls_cache_pending_normal_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("/000000.ts")); + assert!(body.contains("/000001.ts")); + assert!(body.contains("/000002.ts")); + assert!(!body.contains("/000003.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!( + session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, + stored_before + ); + assert_eq!(media_uri_count(&stored_before), 6); + } + + #[tokio::test] + async fn hls_cache_idle_normal_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Idle, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("idle normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("/000000.ts")); + assert!(body.contains("/000001.ts")); + assert!(body.contains("/000002.ts")); + assert!(!body.contains("/000003.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!( + session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, + stored_before + ); + assert_eq!(media_uri_count(&stored_before), 6); + } + + #[tokio::test] + async fn hls_cache_activated_normal_manifest_skips_initial_strip() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 6); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + } + + #[tokio::test] + async fn hls_cache_fresh_required_normal_manifest_does_not_serve_stale_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let rendered_at_ms = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + rendered_at_ms + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO).requiring_newer_manifest(rendered_at_ms), + ) + .await; + + assert!(response.is_none()); + } + + #[tokio::test] + async fn hls_cache_fresh_required_normal_manifest_waits_for_newer_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let (proxy_session_id, old_rendered_at_ms) = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + let old_rendered_at_ms = super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), old_rendered_at_ms); + session.origin_refresh.in_flight = true; + session.mark_authorized_media_access(old_rendered_at_ms); + (proxy_session_id, old_rendered_at_ms) + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 100, + last_proxy_seq: 100, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + segment_proxy_seqs: vec![100], + }); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)) + .requiring_newer_manifest(old_rendered_at_ms), + ) + .await + .expect("fresh manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert!(body.contains("000100.ts")); + assert!(!body.contains("000000.ts")); + assert!(body.contains(&access_lease_id.0)); + } + + #[tokio::test] + async fn hls_cache_pending_transient_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest(transient_manifest_body(&proxy_session_id), 100); + session.mark_authorized_media_access(super::current_time_millis()); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + assert_eq!( + session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), + &stored_before + ); + assert_eq!(media_uri_count(&stored_before), 6); + } + + #[tokio::test] + async fn hls_cache_idle_transient_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest(transient_manifest_body(&proxy_session_id), 100); + session.mark_authorized_media_access(super::current_time_millis()); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Idle, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("idle transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!( + session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), + &stored_before + ); + assert_eq!(media_uri_count(&stored_before), 6); + } + + #[tokio::test] + async fn hls_cache_activated_transient_manifest_skips_initial_strip() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest(transient_manifest_body(&proxy_session_id), 100); + session.mark_authorized_media_access(super::current_time_millis().saturating_sub(16_000)); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 6); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + } + + #[tokio::test] + async fn hls_cache_transient_manifest_without_media_activity_is_not_served_from_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + session + .transient + .replace_manifest_with_validity(transient_manifest_body(&proxy_session_id), rendered_at_ms, 60_000); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); + } + + #[tokio::test] + async fn hls_cache_no_media_yet_transient_manifest_is_served_for_initial_canonical_response() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::current_time_millis(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_id), + rendered_at_ms, + 60_000, + ); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("initial transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + } + + #[tokio::test] + async fn hls_cache_transient_manifest_outside_soft_window_is_not_served_from_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest(transient_manifest_body(&proxy_session_id), 100); + session.mark_authorized_media_access(super::current_time_millis().saturating_sub(60_000)); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); + } + + #[tokio::test] + async fn hls_cache_expired_transient_manifest_with_active_binding_is_served_while_manifest_valid() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let now_ms = super::current_time_millis(); + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.clone(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_id.0), + now_ms.saturating_sub(1_000), + 60_000, + ); + session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.origin_refresh.in_flight = true; + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("valid committed transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + } + + #[tokio::test] + async fn hls_cache_expired_transient_manifest_with_active_binding_is_not_served_after_manifest_validity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let now_ms = super::current_time_millis(); + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.clone(); + session.transient.replace_manifest_with_validity( + transient_manifest_body(&proxy_session_id.0), + now_ms.saturating_sub(60_000), + 1_000, + ); + session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); + } + + #[tokio::test] + async fn hls_cache_no_media_yet_waits_for_first_normal_manifest_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 100, + last_proxy_seq: 100, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + segment_proxy_seqs: vec![100], + }); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("initial normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + } + + #[tokio::test] + async fn hls_cache_no_media_yet_waits_for_first_transient_manifest_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session + .transient + .replace_manifest_with_validity(transient_manifest_body(&proxy_session_for_body), rendered_at_ms, 60_000); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("initial transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + } + + #[tokio::test] + async fn hls_cache_expired_transient_manifest_waits_for_revalidation_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.mark_authorized_media_access(super::current_time_millis().saturating_sub(60_000)); + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::current_time_millis(); + session + .transient + .replace_manifest_with_validity(transient_manifest_body(&proxy_session_for_body), rendered_at_ms, 60_000); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("revalidated transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + } + + async fn grant_hls_proxy_lease(app_state: &Arc, proxy_session_id: &str) -> String { + create_active_hls_user_session(app_state).await; + let now_ms = super::current_time_millis(); + let lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); + let family_key = HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key); + app_state + .hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + family_key, + ProxySessionId(proxy_session_id.to_string()), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + super::hls_access_lease_ttl_ms(app_state), + )) + .await; + assert!(app_state + .hls_proxy + .activate_access_lease( + &lease_id, + &ProxySessionId(proxy_session_id.to_string()), + now_ms, + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::hls_access_lease_ttl_ms(app_state), + }, + ) + .await + .is_activated()); + lease_id.0 + } + + async fn hls_proxy_uri(app_state: &Arc, proxy_session_id: &str, suffix: &str) -> String { + let access_lease_id = grant_hls_proxy_lease(app_state, proxy_session_id).await; + format!("/hls/shared/live/{proxy_session_id}/{access_lease_id}/{suffix}") + } + + async fn wait_for_provider_connection_count(app_state: &Arc, expected: usize) { + for _ in 0..50 { + let actual = app_state.active_provider.get_provider_connections_count().await; + if actual == expected { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(app_state.active_provider.get_provider_connections_count().await, expected); + } + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, "http://origin.example.com/live/final/index.m3u8") { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + async fn map_segment(app_state: &Arc, proxy_seq: u64, extension: &str) -> String { + map_segment_with_origin_url(app_state, proxy_seq, extension, &format!("{proxy_seq}.{extension}")).await + } + + async fn map_segment_with_origin_url( + app_state: &Arc, + proxy_seq: u64, + _extension: &str, + origin_url: &str, + ) -> String { + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let manifest = + normal_manifest(&format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{proxy_seq}\n#EXTINF:4.0,\n{origin_url}\n")); + let mut session = session.write().await; + session.proxy_next_seq = Some(proxy_seq); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() + } + + async fn map_ready_segment(app_state: &Arc, proxy_seq: u64, extension: &str, body: &[u8]) -> String { + let proxy_session_id = map_ready_segment_without_lease(app_state, proxy_seq, extension, body).await; + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + proxy_session_id + } + + async fn map_ready_segment_without_lease( + app_state: &Arc, + proxy_seq: u64, + extension: &str, + body: &[u8], + ) -> String { + let proxy_session_id = map_segment(app_state, proxy_seq, extension).await; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + let cache_key = { + let session = session.read().await; + session.segments.get(&proxy_seq).expect("segment should be mapped").cache_key.clone() + }; + let metadata = app_state + .hls_proxy + .segment_cache() + .write_bytes_and_commit(&cache_key, body) + .await + .expect("cache commit should succeed"); + { + let mut session = session.write().await; + session.segments.get_mut(&proxy_seq).expect("segment should be mapped").status = + SegmentCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; + } + proxy_session_id + } + + async fn map_hls_map(app_state: &Arc, body: &[u8], grant_lease: bool) -> String { + let session = app_state + .hls_proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); + let proxy_session_id = { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() + }; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + let cache_key = { + let session = session.read().await; + session.maps.get(&ProxyMapId(0)).expect("map should be mapped").cache_key.clone() + }; + let metadata = app_state + .hls_proxy + .segment_cache() + .write_bytes_and_commit(&cache_key, body) + .await + .expect("map cache commit should succeed"); + { + let mut session = session.write().await; + session.maps.get_mut(&ProxyMapId(0)).expect("map should be mapped").status = + MapCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; + } + if grant_lease { + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + } + proxy_session_id + } + + async fn map_transient_resource( + app_state: &Arc, + origin_url: &str, + extension: &str, + grant_lease: bool, + ) -> (String, String) { + map_transient_resource_with_kind(app_state, origin_url, extension, grant_lease, TransientResourceKind::Segment) + .await + } + + async fn map_transient_resource_with_kind( + app_state: &Arc, + origin_url: &str, + extension: &str, + grant_lease: bool, + kind: TransientResourceKind, + ) -> (String, String) { + let secret = b"rewrite-secret"; + let now_ms = super::current_time_millis(); + let session = app_state.hls_proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), secret, now_ms).await; + let resource_id = build_transient_resource_id(origin_url, secret); + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.transient.upsert_resources([TransientResourceRef::new( + kind, + origin_url, + secret, + now_ms, + 300_000, + Some(extension.to_string()), + )]); + session.proxy_session_id.0.clone() + }; + if grant_lease { + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + } + (proxy_session_id, resource_id.0) + } + + async fn get_response(app_state: Arc, uri: &str, range: Option<&str>) -> Response { + let router = hls_api_register().with_state(app_state); + let mut request = Request::builder().uri(uri); + if let Some(range) = range { + request = request.header(header::RANGE, range); + } + let mut request = request.body(Body::empty()).expect("request should build"); + request.extensions_mut().insert(ConnectInfo(test_addr())); + router.oneshot(request).await.expect("response") + } + + async fn get_status(app_state: Arc, uri: &str) -> StatusCode { + get_response(app_state, uri, None).await.status() + } + + async fn hls_session_last_media_at_ms(app_state: &Arc, proxy_session_id: &str) -> Option { + app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.to_string())) + .await + .expect("session should exist") + .read() + .await + .activity + .last_authorized_media_at_ms + } + + async fn assert_no_hls_cache_stream_registered(app_state: &Arc) { + assert!(app_state.active_users.active_streams().await.is_empty()); + } + + async fn response_body(response: Response) -> bytes::Bytes { + response.into_body().collect().await.expect("body should collect").to_bytes() + } + + fn access_lease_id_from_redirect_location(location: &str) -> &str { + location.trim_end_matches("/manifest.m3u8").rsplit('/').next().expect("access lease id in redirect") + } + + fn proxy_session_id_from_redirect_location(location: &str) -> &str { + let mut parts = location.trim_end_matches("/manifest.m3u8").rsplit('/'); + let _access_lease_id = parts.next().expect("access lease id in redirect"); + parts.next().expect("proxy session id in redirect") + } + + async fn access_lease_session_token( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + ) -> String { + app_state + .hls_proxy + .access_lease(access_lease_id, proxy_session_id, super::current_time_millis()) + .await + .expect("access lease should exist") + .user_session_token + } + + #[test] + fn transient_full_object_cacheable_request_accepts_open_zero_range() { + use crate::api::model::is_hls_transient_full_object_cacheable_request; + + assert!(is_hls_transient_full_object_cacheable_request(None)); + assert!(is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=4-")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=-4")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-1,4-5")))); + } + + async fn assert_hls_cache_stream_registered(app_state: &Arc, proxy_session_id: &str) { + let streams = app_state.active_users.active_streams().await; + assert_eq!(streams.len(), 1); + let stream = &streams[0]; + assert_eq!(stream.username, "hls-user"); + assert_eq!(stream.session_token.as_deref(), Some("hls-session-token")); + assert_eq!(stream.provider.as_ref(), "origin-provider"); + assert_eq!(stream.channel.item_type, PlaylistItemType::LiveHls); + assert!(stream.channel.shared); + assert_eq!( + stream.channel.shared_stream_id, + Some(super::hls_cache_shared_stream_id(&ProxySessionId(proxy_session_id.to_string()))) + ); + assert_eq!(stream.channel.shared_joined_existing, Some(false)); + assert_eq!(stream.channel.url.as_ref(), format!("/hls/shared/live/{proxy_session_id}/manifest.m3u8")); + assert!(!stream.channel.url.contains("test-access-lease")); + assert!(!stream.channel.url.contains("hls-session-token")); + assert!(!stream.channel.url.contains("origin.example.com")); + assert!(!stream.channel.url.contains("/hls/hls-user/")); + } + + fn stats_provider_test_user_session(provider: &str) -> UserSession { + UserSession { + token: "stats-session-token".to_string(), + transition_version: 0, + virtual_id: 12345, + provider: Arc::from(provider), + stream_url: Arc::from("http://origin.example.com/live/12345.m3u8"), + provider_session_headers: HashMap::new(), + addr: test_addr(), + socket_bound: false, + active_addrs: Vec::new(), + ts: 100, + started_at: 100, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + lifecycle: PlaybackLifecycle::Active, + } + } + + #[test] + fn hls_cache_stats_provider_prefers_active_origin_account_binding() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let proxy_session_id = ProxySessionId("stats-session".to_string()); + let binding = HlsOriginAccountBinding::new( + Arc::clone(&origin_source.input_name), + Arc::from("cdn-dev-alias"), + &proxy_session_id, + 100, + ); + let user_session = stats_provider_test_user_session("cdn-dev"); + + let provider = super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); + + assert_eq!(provider.as_ref(), "cdn-dev-alias"); + } + + #[test] + fn hls_cache_stats_provider_falls_back_when_origin_account_binding_is_not_active() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let proxy_session_id = ProxySessionId("stats-session".to_string()); + let mut binding = HlsOriginAccountBinding::new( + Arc::clone(&origin_source.input_name), + Arc::from("cdn-dev-alias"), + &proxy_session_id, + 100, + ); + binding.detach(HlsOriginAccountDetachedReason::Cleanup, 200); + let user_session = stats_provider_test_user_session("session-provider"); + + let provider = super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); + + assert_eq!(provider.as_ref(), "session-provider"); + } + + #[test] + fn hls_cache_stats_provider_falls_back_to_input_name_without_session_provider() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let user_session = stats_provider_test_user_session(""); + + let provider = super::hls_cache_stats_provider(&origin_source, None, &user_session); + + assert_eq!(provider.as_ref(), "cdn-dev"); + } + + async fn register_hls_cache_stream_for_stats_test( + app_state: &Arc, + session: &crate::api::model::HlsSessionHandle, + proxy_session_id: &ProxySessionId, + session_token: &str, + fingerprint: &Fingerprint, + lease_id: &str, + ) { + create_active_hls_user_session_with( + app_state, + session_token, + "origin-provider", + "http://origin.example.com/live/12345.m3u8", + fingerprint.addr, + ) + .await; + let context = test_hls_access_context_with( + proxy_session_id.clone(), + HlsAccessLeaseId(lease_id.to_string()), + session_token, + fingerprint.key.clone(), + ); + super::ensure_hls_cache_stream_registered(app_state, fingerprint, &HeaderMap::new(), &context, session) + .await + .expect("HLS stream registers"); + } + + fn find_stream_by_session_token( + streams: &[shared::model::StreamInfo], + session_token: &str, + ) -> shared::model::StreamInfo { + streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(session_token)) + .unwrap_or_else(|| panic!("{session_token} stream should exist")) + .clone() + } + + #[tokio::test] + async fn hls_cache_stream_stats_mark_additional_viewers_as_joined_existing() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + let proxy_session_id = ProxySessionId(proxy_session_id); + let shared_stream_id = super::hls_cache_shared_stream_id(&proxy_session_id); + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("session should exist"); + let first_fingerprint = test_fingerprint(); + let second_fingerprint = test_fingerprint_with_addr(test_addr_with_port(55124)); + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-session-token", + &first_fingerprint, + "first-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + let first_meter_uid = first_stream.meter_uid; + assert!(first_stream.channel.shared); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-second-session-token", + &second_fingerprint, + "second-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + let second_stream = find_stream_by_session_token(&streams, "hls-second-session-token"); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); + + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-session-token", + &first_fingerprint, + "first-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(first_stream.meter_uid, first_meter_uid); + } + + struct TestSegmentOrigin { + base_url: String, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for TestSegmentOrigin { + fn drop(&mut self) { self.task.abort(); } + } + + async fn spawn_test_segment_origin(body: &'static [u8]) -> TestSegmentOrigin { + spawn_test_status_origin(StatusCode::OK, body).await + } + + async fn spawn_test_status_origin(status: StatusCode, body: &'static [u8]) -> TestSegmentOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let reason = status.canonical_reason().unwrap_or("Status"); + let response = format!( + "HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + status.as_u16(), + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.write_all(body).await; + }); + } + }); + TestSegmentOrigin { base_url: format!("http://{addr}"), task } + } + + async fn wait_for_hls_test_session(app_state: &Arc, session_key: &HlsSessionKey) -> HlsSessionHandle { + for _ in 0..50 { + if let Some(session) = app_state.hls_proxy.sessions().get_by_key(session_key).await { + return session; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("shared hls test session should be created"); + } + + async fn wait_for_hls_refresh_in_flight(session: &HlsSessionHandle) { + for _ in 0..50 { + if session.read().await.origin_refresh.in_flight { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("shared hls refresh should be in flight"); + } + + struct TestTransientOrigin { + base_url: String, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for TestTransientOrigin { + fn drop(&mut self) { self.task.abort(); } + } + + async fn spawn_test_transient_origin() -> TestTransientOrigin { + spawn_test_transient_origin_with_response( + "206 Partial Content", + &[ + ("Content-Type", "video/mp2t"), + ("Content-Range", "bytes 2-15/16"), + ("Accept-Ranges", "bytes"), + ("Cache-Control", "no-store"), + ("ETag", "\"abc\""), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ], + "transient-body", + ) + .await + } + + async fn spawn_test_transient_origin_with_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: &'static str, + ) -> TestTransientOrigin { + spawn_test_transient_origin_with_delayed_response(status_line, response_headers, body, Duration::ZERO).await + } + + async fn spawn_test_transient_origin_with_delayed_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: &'static str, + response_delay: Duration, + ) -> TestTransientOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + tokio::spawn(async move { + let mut buf = vec![0_u8; 4096]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + requests.lock().await.push(request); + if !response_delay.is_zero() { + tokio::time::sleep(response_delay).await; + } + let mut response = format!("HTTP/1.1 {status_line}\r\nContent-Length: {}\r\n", body.len()); + for (name, value) in response_headers { + let _ = writeln!(&mut response, "{name}: {value}\r"); + } + response.push_str("Connection: close\r\n\r\n"); + response.push_str(body); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + TestTransientOrigin { base_url: format!("http://{addr}"), requests, task } + } + + #[tokio::test] + async fn valid_hls_proxy_segment_without_session_returns_not_found() { + let status = get_status(test_app_state(), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/000123.ts").await; + + assert_eq!(status, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn valid_hls_proxy_segment_with_not_ready_session_returns_not_found() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn valid_hls_proxy_segment_with_not_ready_and_valid_lease_returns_service_unavailable() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn not_ready_hls_proxy_segment_with_fetch_ref_demand_fetches_and_returns_ok() { + let origin = spawn_test_segment_origin(b"0123456789").await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = + map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + } + + #[tokio::test] + async fn not_ready_hls_proxy_segment_with_range_waits_for_demand_fetch_then_returns_partial() { + let origin = spawn_test_segment_origin(b"0123456789").await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = + map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); + } + + #[tokio::test] + async fn not_ready_hls_proxy_segment_without_fetch_ref_returns_service_unavailable() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn ready_hls_proxy_segment_without_lease_returns_not_found() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn invalid_hls_proxy_segment_file_with_valid_lease_redirects_to_channel_unavailable_manifest() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "broken.ts").await; + + let response = get_response(app_state, &uri, None).await; + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_marked_for_gc_redirects_to_expired_manifest() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.mark_for_gc_removal(); + + let response = get_response(app_state, &uri, None).await; + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/hls_session_or_lease_expired.m3u8")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_without_range_returns_ok() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "public, max-age=300, immutable"); + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + { + let session = session.read().await; + let segment = session.segments.get(&123).expect("segment should exist"); + assert_eq!(segment.access.active_readers(), 1); + assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 10, .. })); + } + + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + + { + let session = session.read().await; + let segment = session.segments.get(&123).expect("segment should exist"); + assert_eq!(segment.access.active_readers(), 0); + assert!(segment.access.last_accessed_at_ms() > 0); + assert!(session.activity.last_authorized_media_at_ms.is_some()); + } + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; + } + + #[tokio::test] + async fn ready_hls_proxy_segment_range_zero_open_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "m4s", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4s").await; + + let response = get_response(app_state, &uri, Some("bytes=0-")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-9/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_range_start_open_returns_partial_content_from_offset() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "m4v", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4v").await; + + let response = get_response(app_state, &uri, Some("bytes=4-")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 4-9/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "6"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"456789")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_range_start_end_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_suffix_range_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=-3")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 7-9/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"789")); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_unsatisfiable_range_returns_416() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=99-")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn ready_hls_proxy_segment_multi_range_returns_416() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn ready_hls_proxy_map_without_lease_returns_not_found() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"map-body", false).await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/map/000000.mp4")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn ready_hls_proxy_map_without_range_returns_ok() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; + } + + #[tokio::test] + async fn ready_hls_proxy_map_range_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); + } + + #[tokio::test] + async fn ready_hls_proxy_map_multi_range_returns_416_with_zero_content_length() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn hls_proxy_map_not_ready_with_valid_lease_returns_service_unavailable() { + let app_state = test_app_state(); + let session = + app_state.hls_proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), b"rewrite-secret", 100).await; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); + let proxy_session_id = { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() + }; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn transient_resource_without_lease_returns_not_found() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", false).await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/r/{resource_id}.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn transient_resource_with_valid_lease_streams_origin_response_and_headers() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&crate::api::model::ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + session.origin_request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + session + .origin_request_headers + .insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + session.origin_request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + session + .origin_request_headers + .insert(HeaderName::from_static("x-tuliprox-main-revision"), HeaderValue::from_static("secret")); + session.origin_request_headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + } + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-15/16"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::ETAG], "\"abc\""); + assert_eq!(response.headers()[header::LAST_MODIFIED], "Wed, 21 Oct 2015 07:28:00 GMT"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + let origin_requests = origin.requests.lock().await; + let origin_request = origin_requests.first().expect("origin request").to_ascii_lowercase(); + assert!(origin_request.contains("range: bytes=2-15")); + assert!(origin_request.contains("accept-language: de")); + assert!(!origin_request.contains("authorization: bearer secret")); + assert!(!origin_request.contains("cookie: sid=secret")); + assert!(!origin_request.contains("proxy-authorization: basic secret")); + assert!(!origin_request.contains("host: proxy.example.com")); + assert!(!origin_request.contains("x-tuliprox-main-revision")); + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; + } + + #[tokio::test] + async fn transient_resource_without_range_is_cached_after_first_fetch() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, None).await; + let second = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(second.status(), StatusCode::OK); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(origin.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn transient_resource_range_from_zero_is_cached_as_full_object() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=4-")).await; + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"456789")); + assert_eq!(origin.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn transient_resource_range_from_zero_waits_for_inflight_object_cache_fetch() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_delayed_response( + "200 OK", + &[("Content-Type", "video/mp2t")], + "0123456789", + Duration::from_millis(150), + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first_app_state = Arc::clone(&app_state); + let first_uri = uri.clone(); + let first = tokio::spawn(async move { get_response(first_app_state, &first_uri, Some("bytes=0-")).await }); + for _ in 0..50 { + if origin.requests.lock().await.len() == 1 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(origin.requests.lock().await.len(), 1); + + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; + let first = first.await.expect("first request joins"); + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(origin.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn transient_resource_range_from_offset_without_ready_object_is_not_cached() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(origin.requests.lock().await.len(), 2); + } + + #[tokio::test] + async fn transient_key_resource_is_not_cached() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_response( + "200 OK", + &[("Content-Type", "application/octet-stream")], + "key-bytes", + ) + .await; + let (proxy_session_id, resource_id) = map_transient_resource_with_kind( + &app_state, + &format!("{}/key.bin", origin.base_url), + "key", + true, + TransientResourceKind::Key, + ) + .await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.key")).await; + + let first = get_response(Arc::clone(&app_state), &uri, None).await; + let second = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(second.status(), StatusCode::OK); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"key-bytes")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"key-bytes")); + assert_eq!(origin.requests.lock().await.len(), 2); + } + + #[tokio::test] + async fn transient_resource_origin_error_does_not_mark_media_activity() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_response( + "500 Internal Server Error", + &[("Content-Type", "text/plain")], + "origin-error", + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn transient_resource_permanent_origin_error_redirects_to_channel_unavailable_custom_response() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing") + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn transient_resource_permanent_origin_error_returns_not_found_when_custom_response_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + let origin = + spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing") + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn transient_resource_holds_provider_handle_until_origin_body_is_finished() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); + let session = app_state + .hls_proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id_value) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-a"), + &proxy_session_id_value, + super::current_time_millis(), + )); + } + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); + wait_for_provider_connection_count(&app_state, 0).await; + } + + #[tokio::test] + async fn transient_unknown_resource_redirects_to_channel_unavailable_custom_response() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, _) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(location.ends_with("/cvs/hls/hls-user/hls-pass/channel_unavailable.m3u8")); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[tokio::test] + async fn transient_unknown_resource_returns_not_found_when_custom_response_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, _) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; + } + + #[test] + fn transient_cross_origin_redirect_strips_sensitive_headers() { + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("session=secret")); + headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-1")); + + crate::api::model::scrub_hls_origin_headers(&mut headers, None); + + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key("proxy-authorization")); + assert!(!headers.contains_key(header::HOST)); + assert_eq!(headers[header::RANGE], "bytes=0-1"); + } + + #[tokio::test] + async fn ready_hls_proxy_segment_unknown_range_unit_is_ignored() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("items=0-1")).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + } + + #[tokio::test] + async fn invalid_hls_proxy_file_names_return_not_found() { + let app_state = test_app_state(); + + assert_eq!( + get_status(Arc::clone(&app_state), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/123.ts").await, + StatusCode::NOT_FOUND + ); + assert_eq!( + get_status(app_state, "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/map/000123.exe").await, + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn legacy_hls_route_remains_registered() { + let status = get_status(test_app_state(), "/hls/user/pass/1/2/3/not-a-token").await; + + assert_ne!(status, StatusCode::NOT_FOUND); + } } diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index 07d582f5e..8cd6c714f 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -4,19 +4,20 @@ use crate::{ admission_failure_response, create_m3u_catchup_session_key, create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, is_seek_request, is_session_based_playback, - is_stream_share_enabled, local_stream_response, - redirect, redirect_response, resource_response, + is_stream_share_enabled, local_stream_response, redirect, redirect_response, resource_response, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, - try_option_bad_request, try_result_bad_request, try_result_not_found, - try_unwrap_body, RedirectParams, + try_option_bad_request, try_result_bad_request, try_result_not_found, try_unwrap_body, RedirectParams, }, endpoints::{ - hls_api::{handle_hls_stream_request, m3u_archive_epg_reference_ts}, + hls_api::{ + build_virtual_hls_entry_path, handle_hls_stream_request, hls_admission_failure_manifest_response, + hls_custom_video_manifest_response, m3u_archive_epg_reference_ts, + }, xtream_api::{ApiStreamContext, ApiStreamRequest}, }, model::{AppState, UserApiRequest, UserApiRequestQueryOrBody}, }, - auth::Fingerprint, + auth::{check_network_access_only, resolve_api_user_context, ApiUserAuthError, Fingerprint}, model::{ConfigTarget, ProxyUserCredentials}, repository::{m3u_get_item_for_stream_id, m3u_load_rewrite_playlist, storage_const}, utils::{debug_if_enabled, decode_m3u_catchup_token, has_m3u_catchup_marker, resolve_m3u_catchup_url, M3uCatchupToken, PROVIDER_SCHEME_PREFIX}, @@ -26,14 +27,16 @@ use bytes::Bytes; use futures::StreamExt; use log::{debug, error}; use shared::error::TuliproxError; -use shared::model::ConnectFailureReason; use shared::{ - model::{FieldGetAccessor, PlaylistEntry, PlaylistItemType, TargetType, UserConnectionPermission, XtreamCluster}, + model::{ + ConnectFailureReason, FieldGetAccessor, PlaylistEntry, PlaylistItemType, TargetType, UserConnectionPermission, + XtreamCluster, + }, utils::{concat_path, extract_extension_from_url, sanitize_sensitive_info}, + defaults::{HLS_EXT} }; use std::borrow::Cow; use std::sync::Arc; -use crate::auth::{check_network_access_only, resolve_api_user_context, ApiUserAuthError}; async fn m3u_api( user: Arc, @@ -54,7 +57,8 @@ async fn m3u_api( .status(axum::http::StatusCode::OK) .header(axum::http::header::CONTENT_TYPE, mime::TEXT_PLAIN_UTF_8.to_string()); if content_type == "m3u_plus" { - builder = builder.header(axum::http::header::CONTENT_DISPOSITION, "attachment; filename=\"playlist.m3u\""); + builder = + builder.header(axum::http::header::CONTENT_DISPOSITION, "attachment; filename=\"playlist.m3u\""); } try_unwrap_body!(builder.body(axum::body::Body::from_stream(content_stream))) } @@ -70,8 +74,7 @@ fn m3u_api_with_auth( app_state: &Arc, api_req: &UserApiRequest, ) -> Result<(Arc, Arc), ApiUserAuthError> { - let (user, target) = get_user_target(api_req, app_state) - .ok_or(ApiUserAuthError::AuthFailed)?; + let (user, target) = get_user_target(api_req, app_state).ok_or(ApiUserAuthError::AuthFailed)?; check_network_access_only(&user, fingerprint, app_state)?; Ok((user, target)) } @@ -134,6 +137,25 @@ pub(in crate::api) async fn m3u_api_stream_loaded( return axum::http::StatusCode::BAD_REQUEST.into_response(); } + let is_hls_manifest_request = + stream_ext == Some(HLS_EXT) || (stream_ext.is_none() && extract_extension_from_url(&pli.url) == Some(HLS_EXT)); + + if !user.allows_item_type(pli.item_type) { + if is_hls_manifest_request { + return hls_custom_video_manifest_response( + app_state, + &user, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + axum::http::StatusCode::FORBIDDEN, + ); + } + return crate::api::model::create_custom_video_stream_response( + app_state, + &fingerprint.addr, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); + } let virtual_id = pli.virtual_id; if app_state.active_users.is_user_blocked_for_stream(&user.username, virtual_id).await { @@ -141,6 +163,17 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } if user.permission_denied(app_state) { + if is_hls_manifest_request { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + pli.to_stream_channel(target.id), + pli.input_name.clone(), + req_headers, + ConnectFailureReason::UserAccountExpired, + ); + } return admission_failure_response( app_state, fingerprint, @@ -154,10 +187,8 @@ pub(in crate::api) async fn m3u_api_stream_loaded( if pli.item_type.is_local() { let playback_session_token = create_session_fingerprint(fingerprint, &user.username, virtual_id, false); - let user_session = app_state - .active_users - .get_and_update_user_session(&user.username, &playback_session_token) - .await; + let user_session = + app_state.active_users.get_and_update_user_session(&user.username, &playback_session_token).await; let (admission, _grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( app_state, &user, @@ -205,13 +236,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( archive_discriminator.unwrap_or("live"), ) } else { - create_playback_session_fingerprint( - fingerprint, - &user.username, - virtual_id, - pli.item_type, - Some(extension), - ) + create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, pli.item_type, Some(extension)) }; let eviction_reentry_guard = if pli.item_type == PlaylistItemType::Catchup || !crate::api::api_utils::is_socket_bound_playback_session(pli.item_type, Some(extension)) @@ -224,6 +249,17 @@ pub(in crate::api) async fn m3u_api_stream_loaded( let session_url = if let Some(session) = &user_session { if session.permission == UserConnectionPermission::Exhausted { + if extension == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + pli.to_stream_channel(target.id), + session.provider.clone(), + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, @@ -236,6 +272,17 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } if app_state.active_provider.is_over_limit(&session.provider).await { + if extension == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + pli.to_stream_channel(target.id), + session.provider.clone(), + req_headers, + ConnectFailureReason::ProviderConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, @@ -298,6 +345,17 @@ pub(in crate::api) async fn m3u_api_stream_loaded( session_url.as_ref(), ); if connection_permission == UserConnectionPermission::Exhausted && !allow_exhausted_shared_reconnect { + if extension == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + pli.to_stream_channel(target.id), + input.name.clone(), + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, @@ -337,12 +395,13 @@ pub(in crate::api) async fn m3u_api_stream_loaded( // falling back to `now`. let archive_reference = m3u_archive_epg_reference_ts(&pli.url); // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH - if is_session_request && extension == shared::utils::HLS_EXT { + if is_session_request && extension == shared::defaults::HLS_EXT { + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); return handle_hls_stream_request( fingerprint, app_state, &user, - target.id, + &target, user_session.as_ref(), &pli.url, archive_reference, @@ -350,16 +409,15 @@ pub(in crate::api) async fn m3u_api_stream_loaded( &input, req_headers, connection_permission, - connection_kind, + Some(connection_kind), + &original_hls_entry_path, ) .await .into_response(); } - let pinned_provider = user_session - .as_ref() - .filter(|_| pli.item_type.requires_provider_affinity()) - .map(|session| &session.provider); + let pinned_provider = + user_session.as_ref().filter(|_| pli.item_type.requires_provider_affinity()).map(|session| &session.provider); stream_response( fingerprint, @@ -438,11 +496,21 @@ async fn m3u_api_stream( let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let pli = try_result_not_found!( - m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await, - true, - format!("Failed to read m3u item for stream id {req_virtual_id}") - ); + let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { + Ok(pli) => pli, + Err(err) => { + error!("Failed to read m3u item for stream id {req_virtual_id}: {err}"); + if stream_ext == Some(HLS_EXT) { + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + return crate::api::model::create_custom_video_stream_response( + app_state, + &fingerprint.addr, + crate::api::model::CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); + } + }; let input = try_option_bad_request!( app_state.app_config.get_input_by_name(&pli.input_name), @@ -460,15 +528,6 @@ async fn m3u_api_stream( } else { (pli, None) }; - if !resolved_m3u_item_is_allowed(user.as_ref(), resolved_pli.item_type) { - return crate::api::model::create_custom_video_stream_response( - app_state, - &fingerprint.addr, - crate::api::model::CustomVideoStreamType::ChannelUnavailable, - ) - .into_response(); - } - m3u_api_stream_loaded( user, target, @@ -558,8 +617,8 @@ fn m3u_api_resource_auth( username: &str, password: &str, ) -> Result<(Arc, Arc), ApiUserAuthError> { - let (user, target) = get_user_target_by_credentials(username, password, api_req, app_state) - .ok_or(ApiUserAuthError::AuthFailed)?; + let (user, target) = + get_user_target_by_credentials(username, password, api_req, app_state).ok_or(ApiUserAuthError::AuthFailed)?; resolve_api_user_context(user.clone(), target.clone(), fingerprint.clone(), app_state)?; Ok((user, target)) } diff --git a/backend/src/api/endpoints/mod.rs b/backend/src/api/endpoints/mod.rs index 85973cb2b..05651b3d3 100644 --- a/backend/src/api/endpoints/mod.rs +++ b/backend/src/api/endpoints/mod.rs @@ -18,3 +18,4 @@ pub(in crate::api) mod web_index; pub(in crate::api) mod websocket_api; pub(in crate::api) mod xmltv_api; pub(in crate::api) mod xtream_api; +pub(in crate::api) mod xtream_url; diff --git a/backend/src/api/endpoints/rbac_api.rs b/backend/src/api/endpoints/rbac_api.rs index 5c370b82b..5b1e157fc 100644 --- a/backend/src/api/endpoints/rbac_api.rs +++ b/backend/src/api/endpoints/rbac_api.rs @@ -23,6 +23,8 @@ use std::{ path::{Path as FsPath, PathBuf}, sync::Arc, }; +use shared::defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}; +use crate::utils::{get_default_user_file_path, get_default_user_group_file_path}; const RBAC_MUTATION_LOCK: &str = "rbac:mutation"; @@ -204,13 +206,13 @@ fn user_has_admin_group(user: &WebUiUser) -> bool { user.groups.iter().any(|grou fn count_admin_users(users: &[WebUiUser]) -> usize { users.iter().filter(|user| user_has_admin_group(user)).count() } fn resolve_auth_paths(web_auth: &WebAuthConfig, config_path: &str) -> (PathBuf, PathBuf) { - let userfile_name = if utils::is_blank_or_default_user_file_path(&web_auth.userfile) { - utils::get_default_user_file_path(config_path) + let userfile_name = if is_blank_or_default_user_file_path(&web_auth.userfile) { + get_default_user_file_path(config_path) } else { web_auth.userfile.as_ref().map_or_else(String::new, std::borrow::ToOwned::to_owned) }; - let groupfile_name = if utils::is_blank_or_default_user_group_file_path(&web_auth.groupfile) { - utils::get_default_user_group_file_path(config_path) + let groupfile_name = if is_blank_or_default_user_group_file_path(&web_auth.groupfile) { + get_default_user_group_file_path(config_path) } else { web_auth.groupfile.as_ref().map_or_else(String::new, std::borrow::ToOwned::to_owned) }; @@ -224,12 +226,12 @@ fn resolve_auth_paths(web_auth: &WebAuthConfig, config_path: &str) -> (PathBuf, } }; - let userfile_path = if utils::is_blank_or_default_user_file_path(&web_auth.userfile) { + let userfile_path = if is_blank_or_default_user_file_path(&web_auth.userfile) { PathBuf::from(&userfile_name) } else { resolve_path(&userfile_name) }; - let groupfile_path = if utils::is_blank_or_default_user_group_file_path(&web_auth.groupfile) { + let groupfile_path = if is_blank_or_default_user_group_file_path(&web_auth.groupfile) { PathBuf::from(&groupfile_name) } else { resolve_path(&groupfile_name) diff --git a/backend/src/api/endpoints/stream_history_api.rs b/backend/src/api/endpoints/stream_history_api.rs index 1f0d96f7c..daace7a3d 100644 --- a/backend/src/api/endpoints/stream_history_api.rs +++ b/backend/src/api/endpoints/stream_history_api.rs @@ -1189,6 +1189,7 @@ mod tests { enabled: false, interval_secs: 300, }), + hls_cache: None, }), ..crate::model::Config::default() }; diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index 8bb69e8dd..2708c8bfe 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -34,8 +34,8 @@ fn create_config_input_for_m3u(url: &str) -> ConfigInput { enabled: true, options: Some(ConfigInputOptions { flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, - resolve_delay: shared::utils::default_resolve_delay_secs(), - probe_delay: shared::utils::default_probe_delay_secs(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), probe_live_interval_hours: 120, resolve_filter: None, probe_filter: None, @@ -55,8 +55,8 @@ fn create_config_input_for_xtream(username: &str, password: &str, host: &str) -> enabled: true, options: Some(ConfigInputOptions { flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, - resolve_delay: shared::utils::default_resolve_delay_secs(), - probe_delay: shared::utils::default_probe_delay_secs(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), probe_live_interval_hours: 120, resolve_filter: None, probe_filter: None, @@ -766,6 +766,7 @@ mod tests { metadata: CancellationToken::new(), qos_aggregation: CancellationToken::new(), downloads: CancellationToken::new(), + hls_cache: CancellationToken::new(), }; let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); let (manual_update_sender, _) = mpsc::channel::(1); @@ -783,6 +784,8 @@ mod tests { downloads: Arc::new(crate::api::model::DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(crate::api::model::HlsProxyManager::new()), + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), active_users, active_provider, connection_manager, diff --git a/backend/src/api/endpoints/websocket_api.rs b/backend/src/api/endpoints/websocket_api.rs index 9dc47573b..22e6744ea 100644 --- a/backend/src/api/endpoints/websocket_api.rs +++ b/backend/src/api/endpoints/websocket_api.rs @@ -15,7 +15,8 @@ use shared::{ Permission, ProtocolHandler, ProtocolHandlerMemory, ProtocolMessage, UserCommand, UserRole, WsCloseCode, PERM_ALL, PROTOCOL_VERSION, ROLE_ADMIN, }, - utils::{concat_path_leading_slash, default_kick_secs}, + utils::{concat_path_leading_slash}, + defaults::{default_kick_secs}, }; use std::{fmt, io, sync::Arc}; diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 40e1e8e0f..050346e97 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -6,17 +6,18 @@ use crate::{ api_utils, api_utils::{ admission_failure_response, create_api_proxy_user, create_catchup_session_key, - create_playback_session_fingerprint, create_session_fingerprint, empty_json_response_as_array, empty_json_response_as_object, - force_provider_stream_response, get_session_reservation_ttl_secs, get_user_target, - get_user_target_by_credentials, internal_server_error, is_seek_request, is_session_based_playback, - is_stream_share_enabled, local_stream_response, - redirect, redirect_response, resource_response, - separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, - try_option_bad_request, try_result_bad_request, try_result_not_found, - try_unwrap_body, RedirectParams, + create_playback_session_fingerprint, create_session_fingerprint, empty_json_response_as_array, + empty_json_response_as_object, force_provider_stream_response, get_session_reservation_ttl_secs, + get_user_target, get_user_target_by_credentials, internal_server_error, is_seek_request, + is_session_based_playback, is_stream_share_enabled, local_stream_response, redirect, redirect_response, + resource_response, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, + try_option_bad_request, try_result_bad_request, try_unwrap_body, RedirectParams, }, endpoints::{ - hls_api::handle_hls_stream_request, + hls_api::{ + build_virtual_hls_entry_path, handle_hls_stream_request, hls_admission_failure_manifest_response, + hls_custom_video_manifest_response, + }, xmltv_api::{get_empty_epg_response, get_epg_path_for_target_by_type, serve_short_epg}, }, model::{ @@ -47,18 +48,18 @@ use futures::{ use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; -use shared::model::ConnectFailureReason; use shared::{ concat_string, error::TuliproxError, model::{ - create_stream_channel_with_type, PlaylistEntry, PlaylistItemType, ProxyType, ShortEpgResultDto, TargetType, - UserConnectionPermission, XtreamCluster, XtreamPlaylistItem, + create_stream_channel_with_type, ConnectFailureReason, PlaylistEntry, PlaylistItemType, ProxyType, + ShortEpgResultDto, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem, }, utils::{ deserialize_as_string, extract_extension_from_url, generate_provider_playlist_uuid, sanitize_sensitive_info, trim_slash, Internable, }, + defaults::{HLS_EXT}, }; use std::{ fmt::{Display, Formatter, Write}, @@ -235,7 +236,9 @@ async fn xtream_player_api_stream( let auth_status = app_state.app_config.get_auth_error_status(); let (user, target) = match user_target { None => { - let Some((user, target)) = get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) else { + let Some((user, target)) = + get_user_target_by_credentials(stream_req.username, stream_req.password, api_req, app_state) + else { return auth_status.into_response(); }; (user, target) @@ -248,25 +251,44 @@ async fn xtream_player_api_stream( } let _guard = app_state.app_config.file_locks.write_lock_str(&user.username).await; + let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); + let is_hls_manifest_request = stream_ext == Some(HLS_EXT); let target_name = &target.name; if !target.has_output(TargetType::Xtream) { debug!("Target has no xtream codes playlist {target_name}"); + if is_hls_manifest_request { + // Preserve plain auth-status behaviour for HLS manifest probes — + // returning an HLS manifest body (even with 404) breaks auth-probes + // and monitoring/observability that assert on the original 401/403. + return auth_status.into_response(); + } return create_custom_video_stream_response( app_state, &fingerprint.addr, CustomVideoStreamType::ChannelUnavailable, ) - .into_response(); + .into_response(); } - let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); - let pli = try_result_not_found!( - xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, None).await, - true, - format!("Failed to read xtream item for stream id {req_virtual_id}") - ); + let Ok(pli) = xtream_get_item_for_stream_id(req_virtual_id, app_state, &target, None).await else { + error!("Failed to read xtream item for stream id {req_virtual_id}"); + if is_hls_manifest_request { + return hls_custom_video_manifest_response( + app_state, + &user, + CustomVideoStreamType::ChannelUnavailable, + axum::http::StatusCode::NOT_FOUND, + ); + } + return create_custom_video_stream_response( + app_state, + &fingerprint.addr, + CustomVideoStreamType::ChannelUnavailable, + ) + .into_response(); + }; let output_allowed = if stream_req.context == ApiStreamContext::Timeshift { user.allows_cluster(XtreamCluster::Live) @@ -274,12 +296,20 @@ async fn xtream_player_api_stream( user.allows_item_type(pli.item_type) }; if !output_allowed { + if is_hls_manifest_request { + return hls_custom_video_manifest_response( + app_state, + &user, + CustomVideoStreamType::ChannelUnavailable, + axum::http::StatusCode::NOT_FOUND, + ); + } return create_custom_video_stream_response( app_state, &fingerprint.addr, CustomVideoStreamType::ChannelUnavailable, ) - .into_response(); + .into_response(); } let virtual_id = pli.virtual_id; @@ -297,11 +327,23 @@ async fn xtream_player_api_stream( ); if user.permission_denied(app_state) { + let stream_channel = create_stream_channel_with_type(target.id, &pli, pli.item_type); + if resolve_xtream_playback_extension(stream_ext, &pli).is_some_and(|ext| ext == HLS_EXT) { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + stream_channel, + pli.input_name.clone(), + req_headers, + ConnectFailureReason::UserAccountExpired, + ); + } return admission_failure_response( app_state, fingerprint, &user, - create_stream_channel_with_type(target.id, &pli, pli.item_type), + stream_channel, pli.input_name.clone(), req_headers, ConnectFailureReason::UserAccountExpired, @@ -310,10 +352,8 @@ async fn xtream_player_api_stream( if pli.item_type.is_local() { let playback_session_token = create_session_fingerprint(fingerprint, &user.username, virtual_id, false); - let user_session = app_state - .active_users - .get_and_update_user_session(&user.username, &playback_session_token) - .await; + let user_session = + app_state.active_users.get_and_update_user_session(&user.username, &playback_session_token).await; let (admission, _grace_mode, request_class) = crate::api::api_utils::resolve_playback_request_admission( app_state, &user, @@ -326,7 +366,7 @@ async fn xtream_player_api_stream( false, false, ) - .await; + .await; return local_stream_response( fingerprint, app_state, @@ -341,8 +381,8 @@ async fn xtream_player_api_stream( Some(request_class), true, ) - .await - .into_response(); + .await + .into_response(); } let resolved_stream_ext = resolve_xtream_playback_extension(stream_ext, &pli); @@ -369,13 +409,7 @@ async fn xtream_player_api_stream( let session_key = if item_type == PlaylistItemType::Catchup { create_catchup_session_key(fingerprint, &user.username, virtual_id) } else { - create_playback_session_fingerprint( - fingerprint, - &user.username, - virtual_id, - item_type, - Some(playback_ext), - ) + create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, item_type, Some(playback_ext)) }; let eviction_reentry_guard = if item_type == PlaylistItemType::Catchup || !crate::api::api_utils::is_socket_bound_playback_session(item_type, Some(playback_ext)) @@ -388,11 +422,23 @@ async fn xtream_player_api_stream( let session_url = if let Some(session) = &user_session { if session.permission == UserConnectionPermission::Exhausted { + let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); + if playback_ext == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + stream_channel, + session.provider.clone(), + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, &user, - create_stream_channel_with_type(target.id, &pli, item_type), + stream_channel, session.provider.clone(), req_headers, ConnectFailureReason::UserConnectionsExhausted, @@ -400,11 +446,23 @@ async fn xtream_player_api_stream( } if app_state.active_provider.is_over_limit(&session.provider).await { + let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); + if playback_ext == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + stream_channel, + session.provider.clone(), + req_headers, + ConnectFailureReason::ProviderConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, &user, - create_stream_channel_with_type(target.id, &pli, item_type), + stream_channel, session.provider.clone(), req_headers, ConnectFailureReason::ProviderConnectionsExhausted, @@ -428,8 +486,8 @@ async fn xtream_player_api_stream( }, None, ) - .await - .into_response(); + .await + .into_response(); } session.stream_url.clone() @@ -449,7 +507,7 @@ async fn xtream_player_api_stream( false, false, ) - .await; + .await; let connection_permission = connection_admission.permission; let connection_kind = connection_admission .kind @@ -462,11 +520,23 @@ async fn xtream_player_api_stream( session_url.as_ref(), ); if connection_permission == UserConnectionPermission::Exhausted && !allow_exhausted_shared_reconnect { + let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); + if playback_ext == HLS_EXT { + return hls_admission_failure_manifest_response( + app_state, + fingerprint, + &user, + stream_channel, + input.name.clone(), + req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ); + } return admission_failure_response( app_state, fingerprint, &user, - create_stream_channel_with_type(target.id, &pli, item_type), + stream_channel, input.name.clone(), req_headers, ConnectFailureReason::UserConnectionsExhausted, @@ -505,12 +575,13 @@ async fn xtream_player_api_stream( let is_session_request = is_session_based_playback(item_type, Some(playback_ext)); // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH - if is_session_request && playback_ext == shared::utils::HLS_EXT { + if is_session_request && playback_ext == shared::defaults::HLS_EXT { + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, pli.virtual_id); return handle_hls_stream_request( fingerprint, app_state, &user, - target.id, + &target, user_session.as_ref(), &stream_url, None, @@ -518,18 +589,17 @@ async fn xtream_player_api_stream( &input, req_headers, connection_permission, - connection_kind, + connection_admission.kind, + &original_hls_entry_path, ) - .await - .into_response(); + .await + .into_response(); } let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); - let pinned_provider = user_session - .as_ref() - .filter(|_| item_type.requires_provider_affinity()) - .map(|session| &session.provider); + let pinned_provider = + user_session.as_ref().filter(|_| item_type.requires_provider_affinity()).map(|session| &session.provider); stream_response( fingerprint, @@ -548,8 +618,8 @@ async fn xtream_player_api_stream( allow_exhausted_shared_reconnect, grace_mode, ) - .await - .into_response() + .await + .into_response() } pub(crate) fn get_query_path( @@ -613,9 +683,7 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( target_id: u16, stream_req: ApiStreamRequest<'_>, ) -> impl IntoResponse + Send { - if stream_req.access_token - && !verify_access_token(stream_req.password, &app_state.app_config.access_token_secret) - { + if stream_req.access_token && !verify_access_token(stream_req.password, &app_state.app_config.access_token_secret) { return axum::http::StatusCode::FORBIDDEN.into_response(); } @@ -660,8 +728,8 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( None, true, ) - .await - .into_response(); + .await + .into_response(); } let requested_extension = resolve_xtream_playback_extension(stream_ext, &pli); @@ -672,23 +740,19 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( if playback_ext.is_empty() { requested_extension.as_deref() } else { Some(&*playback_ext) }; let is_session_request = is_session_based_playback(pli.item_type, playback_ext); - let session_key = create_playback_session_fingerprint( - fingerprint, - "webui", - virtual_id, - pli.item_type, - playback_ext, - ); + let session_key = + create_playback_session_fingerprint(fingerprint, "webui", virtual_id, pli.item_type, playback_ext); // TODO how should we use fixed provider for hls in multi provider config? // Reverse proxy mode — only route genuine HLS into the HLS handler, not DASH - if is_session_request && playback_ext == Some(shared::utils::HLS_EXT) { + if is_session_request && playback_ext == Some(shared::defaults::HLS_EXT) { + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); return handle_hls_stream_request( fingerprint, app_state, &user, - target.id, + &target, None, &pli.url, None, @@ -696,10 +760,11 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( &input, req_headers, UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, + Some(crate::api::model::ConnectionKind::Normal), + &original_hls_entry_path, ) - .await - .into_response(); + .await + .into_response(); } let stream_url = try_option_bad_request!( @@ -729,8 +794,8 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( false, None, ) - .await - .into_response() + .await + .into_response() } else { axum::http::StatusCode::BAD_REQUEST.into_response() } @@ -744,7 +809,9 @@ async fn xtream_player_api_resource( resource_req: ApiStreamRequest<'_>, ) -> impl IntoResponse { let auth_status = app_state.app_config.get_auth_error_status(); - let Some((user, target)) = get_user_target_by_credentials(resource_req.username, resource_req.password, api_req, app_state) else { + let Some((user, target)) = + get_user_target_by_credentials(resource_req.username, resource_req.password, api_req, app_state) + else { return auth_status.into_response(); }; if let Err(e) = check_permission_and_network_access_only(&user, fingerprint, app_state) { @@ -777,7 +844,10 @@ async fn xtream_player_api_resource( let redirect_url = api_utils::resolve_redirect_location(input.as_deref(), &url); match redirect_url { Ok(redirect_url) => { - trace_if_enabled!("Redirecting resource request to {}", sanitize_sensitive_info(redirect_url.as_ref())); + trace_if_enabled!( + "Redirecting resource request to {}", + sanitize_sensitive_info(redirect_url.as_ref()) + ); redirect(redirect_url.as_ref()).into_response() } Err(err) => { @@ -886,7 +956,9 @@ async fn xtream_player_api_timeshift_stream( let api_req = UserApiRequest::merge_prefer_primary(&path_req, &query_req); let auth_status = app_state.app_config.get_auth_error_status(); - let Some((user, target)) = get_user_target_by_credentials(&api_req.username, &api_req.password, &api_req, &app_state) else { + let Some((user, target)) = + get_user_target_by_credentials(&api_req.username, &api_req.password, &api_req, &app_state) + else { return auth_status.into_response(); }; @@ -912,8 +984,8 @@ async fn xtream_player_api_timeshift_stream( ), Some((user, target)), ) - .await - .into_response() + .await + .into_response() } async fn xtream_player_api_timeshift_query_stream( @@ -932,7 +1004,9 @@ async fn xtream_player_api_timeshift_query_stream( } let auth_status = app_state.app_config.get_auth_error_status(); - let Some((user, target)) = get_user_target_by_credentials(&api_req.username, &api_req.password, &api_req, &app_state) else { + let Some((user, target)) = + get_user_target_by_credentials(&api_req.username, &api_req.password, &api_req, &app_state) + else { return auth_status.into_response(); }; @@ -958,8 +1032,8 @@ async fn xtream_player_api_timeshift_query_stream( ), Some((user, target)), ) - .await - .into_response() + .await + .into_response() } #[allow(clippy::too_many_lines)] @@ -1015,7 +1089,6 @@ pub async fn xtream_get_stream_info_response( if pli.provider_id > 0 { if let Some(input) = input { if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, cluster, pli.provider_id) { - // redirect is only possible for live streams if user.proxy == ProxyType::Redirect && cluster == XtreamCluster::Live { return match api_utils::resolve_redirect_location(Some(&input), &info_url) { @@ -1037,7 +1110,9 @@ pub async fn xtream_get_stream_info_response( &pli, info_url.as_str(), cluster, - ).await { + ) + .await + { return try_unwrap_body!(axum::response::Response::builder() .status(axum::http::StatusCode::OK) .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) @@ -1096,7 +1171,7 @@ async fn xtream_get_short_epg( stream_id.intern(), limit, ) - .await; + .await; } } @@ -1130,7 +1205,7 @@ async fn xtream_get_short_epg( None, false, ) - .await + .await { Ok((content, _)) => ( axum::http::StatusCode::OK, @@ -1347,7 +1422,11 @@ macro_rules! skip_flag_optional { } #[allow(clippy::too_many_lines)] -async fn xtream_player_api(fingerprint: &Fingerprint, api_req: UserApiRequest, app_state: &Arc) -> impl IntoResponse + Send { +async fn xtream_player_api( + fingerprint: &Fingerprint, + api_req: UserApiRequest, + app_state: &Arc, +) -> impl IntoResponse + Send { api_req.log_sanitized("xtream_player_api"); let auth_status = app_state.app_config.get_auth_error_status(); let Some((user, target)) = get_user_target(&api_req, app_state) else { @@ -1451,7 +1530,7 @@ async fn xtream_player_api(fingerprint: &Fingerprint, api_req: UserApiRequest, a category_id, &user, ) - .await + .await { return response.into_response(); } @@ -1499,9 +1578,9 @@ async fn xtream_player_api(fingerprint: &Fingerprint, api_req: UserApiRequest, a } } -fn xtream_create_content_stream(xtream_iter: S) -> impl Stream> +fn xtream_create_content_stream(xtream_iter: S) -> impl Stream> where - S: Stream + Send + Unpin + 'static, + S: Stream + Send + Unpin + 'static, { let mapped = xtream_iter.map(move |(mut line, has_next)| { if has_next { @@ -1584,8 +1663,8 @@ async fn xtream_player_token_stream( target_id, ApiStreamRequest::from_access_token(ctxt, &token, &stream_id, ""), ) - .await - .into_response() + .await + .into_response() } pub fn xtream_api_register() -> axum::Router> { @@ -1625,7 +1704,9 @@ mod tests { }; use crate::{api::model::UserApiRequest, model::ConfigInput}; use shared::{ - model::{InputType, PlaylistItemType, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem}, + model::{ + InputType, PlaylistItemType, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, + }, utils::Internable, }; use std::sync::Arc; @@ -1856,7 +1937,8 @@ mod tests { url: "http://pms-user:pms-pass@pms.example.invalid:32400".to_string(), ..ConfigInput::default() }; - let fallback = Arc::::from("media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted%2Ffile.mkv"); + let fallback = + Arc::::from("media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted%2Ffile.mkv"); let resolved = get_xtream_player_api_stream_url(&input, ApiStreamContext::Movie, "813563.mkv", &fallback) .expect("media-server fallback should be preserved"); diff --git a/backend/src/api/endpoints/xtream_url.rs b/backend/src/api/endpoints/xtream_url.rs new file mode 100644 index 000000000..6cc8a6fdc --- /dev/null +++ b/backend/src/api/endpoints/xtream_url.rs @@ -0,0 +1,28 @@ +//! Shared URL-building helpers for the xtream and HLS code paths. +//! +//! Three symbols (`ApiStreamContext`, `get_query_path`, +//! `get_xtream_player_api_stream_url`) historically live in `xtream_api.rs` +//! but both `xtream_api` and `hls_api` need them. Importing them directly from +//! `xtream_api` creates a visible cross-import between two sibling endpoint +//! files, which is the symptom of a missing service layer. Until the full +//! `services::hls::playback` extraction lands (see ARCH-1 roadmap below), +//! route the cross-import through this re-export module so: +//! +//! - `hls_api` no longer depends on a sibling endpoint file directly. +//! - The cross-dependency is one-way and visible at a single import site. +//! - Future extraction only has to move this file's contents, not chase +//! cross-imports across the `hls_api` surface. +//! +//! # ARCH-1 roadmap (deferred) +//! +//! The full `ARCH-1` fix extracts a `services::hls::playback` module that owns +//! the auth, admission, session lookup, lease, cache serve, and fallback +//! pipeline. `hls_api.rs` then becomes a thin Axum handler layer. That +//! extraction touches about 50 imports and 30 functions; it is intentionally +//! deferred from this commit because the change requires concurrent test +//! re-orchestration for every state-machine branch in the cache, and any +//! partial move is more dangerous than a single atomic commit. This +//! re-export module is the smallest step that unblocks the dependency +//! direction without that risk. + +pub(in crate::api) use super::xtream_api::{get_query_path, get_xtream_player_api_stream_url, ApiStreamContext}; \ No newline at end of file diff --git a/backend/src/api/http_layers.rs b/backend/src/api/http_layers.rs index 5faa37262..00bf32892 100644 --- a/backend/src/api/http_layers.rs +++ b/backend/src/api/http_layers.rs @@ -5,5 +5,5 @@ pub(crate) fn create_cors_layer( .allow_origin(tower_http::cors::Any) .allow_methods(methods) .allow_headers(tower_http::cors::Any) - .max_age(std::time::Duration::from_secs(3600)) + .max_age(std::time::Duration::from_hours(1)) } diff --git a/backend/src/api/main_api.rs b/backend/src/api/main_api.rs index 0d884098d..d28be41fa 100644 --- a/backend/src/api/main_api.rs +++ b/backend/src/api/main_api.rs @@ -20,9 +20,10 @@ use crate::{ http_layers::create_cors_layer, model::{ create_cache, create_http_client, create_http_client_no_redirect, exec_provider_dns, - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, DownloadQueue, - EventManager, EventMessage, HdHomerunAppState, ManualPlaylistUpdateRequest, MetadataUpdateManager, PlaylistStorageState, - SharedStreamManager, UpdateGuard, exec_qos_aggregation, + exec_hls_cache_gc, exec_hls_lifecycle, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, + ConnectionManager, DownloadQueue, EventManager, EventMessage, HdHomerunAppState, HlsProvisioningState, + HlsProxyManager, ManualPlaylistUpdateRequest, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, + UpdateGuard, exec_qos_aggregation, }, panel_api::sync_panel_api_exp_dates_on_boot, scheduler::{exec_interner_prune, exec_scheduler}, @@ -284,6 +285,14 @@ async fn create_shared_data( let event_manager = Arc::new(EventManager::new()); let active_provider = Arc::new(ActiveProviderManager::new(app_config, &event_manager)); let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + let rewrite_secret = config + .reverse_proxy + .as_ref() + .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); + let hls_proxy = Arc::new(HlsProxyManager::from_hls_cache_config_and_secret( + config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()), + &rewrite_secret, + )); active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); active_users.start_adaptive_expiry_worker(); @@ -314,6 +323,8 @@ async fn create_shared_data( downloads: Arc::new(DownloadQueue::new_with_state_file(Some(downloads_state_file))), cache: Arc::new(ArcSwapOption::from(cache)), shared_stream_manager, + hls_proxy, + hls_provisioning: Arc::new(HlsProvisioningState::new()), active_users, active_provider, connection_manager, @@ -363,6 +374,7 @@ async fn cancel_all_service_tokens(app_state: &Arc) { cancel_tokens.provider_dns.cancel(); cancel_tokens.qos_aggregation.cancel(); cancel_tokens.downloads.cancel(); + cancel_tokens.hls_cache.cancel(); app_state.active_users.shutdown(); // Use the manager's shutdown() rather than cancelling the token directly so // the is_shutdown flag is set and workers do not attempt to restart after cancellation. @@ -602,6 +614,8 @@ pub async fn start_server(app_config: Arc, targets: Arc) -> Option> { + self.providers.find_provider_config(provider_name) + } + + pub fn is_provider_for_input(&self, provider_name: &Arc, input_name: &Arc) -> bool { + self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) + } + + pub async fn is_provider_reserved_for_other_session( + &self, + provider_name: &Arc, + session_owner: Option<&str>, + ) -> bool { + self.is_reserved_for_other(provider_name, session_owner).await + } + pub async fn active_connections(&self) -> Option, usize>> { self.providers.active_connections().await } @@ -1512,7 +1528,7 @@ mod tests { }; use std::collections::HashSet; use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; - use shared::utils::{default_probe_user_priority, default_user_priority}; + use shared::defaults::{default_probe_user_priority, default_user_priority}; fn build_test_app_config(aliases: Option>, max_connections: u16) -> AppConfig { let input = Arc::new(ConfigInput { diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index 7f5cc7413..baed302b6 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -13,9 +13,13 @@ use shared::{ UserConnectionPermission, VirtualId, }, utils::{ - current_time_secs, default_grace_period_millis, default_grace_period_timeout_secs, default_hls_session_ttl_secs, - extract_extension_from_url, sanitize_sensitive_info, strip_port, Internable, DASH_EXT, HLS_EXT, + current_time_secs, + extract_extension_from_url, sanitize_sensitive_info, strip_port, Internable, }, + defaults::{ + default_grace_period_millis, default_grace_period_timeout_secs, default_hls_session_ttl_secs, + DASH_EXT, HLS_EXT, + } }; use std::{ borrow::Cow, @@ -725,19 +729,23 @@ impl ActiveUserManager { async fn log_active_user(&self) { let is_log_user_enabled = self.is_log_user_enabled(); + // Skip the full connection-map snapshot + event send entirely when logging + // is disabled — this runs on every connection add/release and dominates + // lock contention on the active-user path at high segment rates. + if !is_log_user_enabled { + return; + } let (user_count, user_connection_count) = { self.active_users_and_connections().await }; self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Connections( user_count, user_connection_count, ))); - if is_log_user_enabled { - let last_user_count = self.last_logged_user_count.load(Ordering::Relaxed); - let last_connection_count = self.last_logged_user_connection_count.load(Ordering::Relaxed); - if last_user_count != user_count || last_connection_count != user_connection_count { - self.last_logged_user_count.store(user_count, Ordering::Relaxed); - self.last_logged_user_connection_count.store(user_connection_count, Ordering::Relaxed); - info!("Active Users: {user_count}, Active User Connections: {user_connection_count}"); - } + let last_user_count = self.last_logged_user_count.load(Ordering::Relaxed); + let last_connection_count = self.last_logged_user_connection_count.load(Ordering::Relaxed); + if last_user_count != user_count || last_connection_count != user_connection_count { + self.last_logged_user_count.store(user_count, Ordering::Relaxed); + self.last_logged_user_connection_count.store(user_connection_count, Ordering::Relaxed); + info!("Active Users: {user_count}, Active User Connections: {user_connection_count}"); } } @@ -1238,6 +1246,66 @@ impl ActiveUserManager { .permission } + pub(crate) async fn refresh_session_connection_kind_for_origin_policy( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + session_token: &str, + ) -> Option { + if max_connections == 0 && soft_connections == 0 { + return Some(ConnectionKind::Normal); + } + + let (connection_kind, promotions, divergence_snapshot) = { + let mut connections = self.connections.write().await; + let connection_data = connections.by_key.get_mut(username)?; + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + + let session_index = connection_data.sessions.iter().position(|session| session.token == session_token)?; + + let promotions = Self::promote_counted_soft_session_to_normal_if_available(connection_data, session_token); + let connection_kind = if connection_data.sessions[session_index].lifecycle.is_counted() + || Self::session_has_stream(connection_data, session_token) + { + Some( + connection_data.sessions[session_index] + .connection_kind + .unwrap_or(ConnectionKind::Normal), + ) + } else { + let admission = self.check_connection_admission_with_counts( + username, + connection_data, + connection_data.effective_counts_for_admission(Some(session_token)), + ); + if admission.permission == UserConnectionPermission::Allowed { + if let Some(kind) = admission.kind { + Self::update_session_admission( + &mut connection_data.sessions[session_index], + admission.permission, + Some(kind), + ); + } + admission.kind + } else { + None + } + }; + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + + (connection_kind, promotions, divergence_snapshot) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + for action in promotions { + self.emit_promotion_update(username, action).await; + } + + connection_kind + } + pub(crate) async fn get_eviction_candidates( &self, username: &str, @@ -1642,6 +1710,51 @@ impl ActiveUserManager { promotions } + fn promote_counted_soft_session_to_normal_if_available( + connection_data: &mut UserConnectionData, + session_token: &str, + ) -> Vec { + if connection_data.max_connections > 0 && connection_data.counts.normal >= connection_data.max_connections { + return Vec::new(); + } + + let Some(session_index) = connection_data.sessions.iter().position(|session| { + session.token == session_token + && session.lifecycle.is_counted() + && session.connection_kind == Some(ConnectionKind::Soft) + }) else { + return Vec::new(); + }; + + if connection_data.counts.soft == 0 { + return Vec::new(); + } + + connection_data.counts.normal = connection_data.counts.normal.saturating_add(1); + connection_data.counts.soft = connection_data.counts.soft.saturating_sub(1); + connection_data.sessions[session_index].connection_kind = Some(ConnectionKind::Normal); + Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); + + let mut promotions = Vec::new(); + for stream in connection_data + .streams + .iter() + .filter(|stream| stream.session_token.as_deref() == Some(session_token)) + { + if connection_data.stream_kinds.get(&stream.uid) != Some(&ConnectionKind::Soft) { + continue; + } + let new_priority = connection_data.stream_normal_priorities.get(&stream.uid).copied().unwrap_or_default(); + connection_data.stream_kinds.insert(stream.uid, ConnectionKind::Normal); + promotions.push(PromotionAction { + addr: stream.addr, + uid: stream.uid, + new_priority, + }); + } + promotions + } + fn bump_session_transition_version(session: &mut UserSession) -> u64 { session.transition_version = session.transition_version.saturating_add(1); session.transition_version @@ -1944,6 +2057,56 @@ impl ActiveUserManager { } } + pub async fn release_session_streams_and_counted_reservation( + &self, + username: &str, + session_token: &str, + ) -> bool { + let (connection_changed, user_removed, promotions, divergence_snapshot) = { + let mut user_connections = self.connections.write().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return false; + }; + + let counted_kind = connection_data + .sessions + .iter() + .find(|session| session.token == session_token && session.lifecycle.is_counted()) + .and_then(|session| session.connection_kind); + let (_removed_streams, mut connection_changed) = + connection_data.remove_streams_for_session_and_release_counted(session_token, counted_kind); + Self::clear_session_counted_without_stream(connection_data, session_token); + + if connection_data.connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let promotions = Self::collect_promotions_after_capacity_release(connection_data); + let user_removed = connection_data.connections == 0 + && connection_data.streams.is_empty() + && connection_data.sessions.is_empty(); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + connection_changed |= !promotions.is_empty(); + + (connection_changed, user_removed, promotions, divergence_snapshot) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + + if user_removed { + let mut user_connections = self.connections.write().await; + user_connections.by_key.remove(username); + } + if connection_changed || user_removed { + self.log_active_user().await; + } + for action in promotions { + self.emit_promotion_update(username, action).await; + } + connection_changed || user_removed + } + pub async fn create_user_session(&self, request: CreateUserSessionParams<'_>) -> String { let CreateUserSessionParams { user, @@ -2618,57 +2781,34 @@ impl ActiveUserManager { pub async fn socket_expiry_deadline(&self, addr: &SocketAddr) -> Option { let ttl_secs = self.active_socket_ttl_secs(); - { - let connections = self.connections.read().await; - let registration = connections.key_by_addr.get(addr)?; - if registration.username.is_empty() { - return None; - } - - let has_non_stable_stream = connections - .by_key - .get(®istration.username) - .is_some_and(|connection_data| { - connection_data - .streams - .iter() - .any(|stream| stream.addr == *addr && !is_stable_session_stream(stream)) - }); - - if !has_non_stable_stream { - return Some(registration.ts.saturating_add(ttl_secs)); - } - } - - let mut connections = self.connections.write().await; - let (username_is_empty, has_non_stable_stream) = { - let registration = connections.key_by_addr.get(addr)?; - let username = registration.username.as_str(); - let username_is_empty = username.is_empty(); - let has_non_stable_stream = !username_is_empty - && connections - .by_key - .get(username) - .is_some_and(|connection_data| { - connection_data - .streams - .iter() - .any(|stream| stream.addr == *addr && !is_stable_session_stream(stream)) - }); - (username_is_empty, has_non_stable_stream) - }; - if username_is_empty { + let connections = self.connections.read().await; + let registration = connections.key_by_addr.get(addr)?; + if registration.username.is_empty() { return None; } - let registration = connections.key_by_addr.get_mut(addr)?; - if has_non_stable_stream { - registration.ts = current_time_secs(); - } - Some(registration.ts.saturating_add(ttl_secs)) } + pub async fn touch_socket_activity(&self, addr: &SocketAddr) { + let now = current_time_secs(); + let mut user_connections = self.connections.write().await; + let Some(username) = user_connections.key_by_addr.get_mut(addr).and_then(|registration| { + if registration.username.is_empty() { + None + } else { + registration.ts = now; + Some(registration.username.clone()) + } + }) else { + return; + }; + + if let Some(connection_data) = user_connections.by_key.get_mut(&username) { + connection_data.ts = now; + } + } + pub async fn touch_http_activity(&self, username: &str, token: &str, addr: &SocketAddr) { let now = current_time_secs(); let mut user_connections = self.connections.write().await; @@ -6879,7 +7019,7 @@ mod tests { } #[tokio::test] - async fn socket_expiry_deadline_refreshes_active_vod_streams() { + async fn socket_expiry_deadline_does_not_refresh_active_vod_streams_without_activity() { let config = Config::default(); let geoip = Arc::new(ArcSwapOption::::default()); let event_manager = Arc::new(EventManager::new()); @@ -6940,21 +7080,84 @@ mod tests { let deadline = manager .socket_expiry_deadline(&addr) .await - .expect("continuous VOD streams should stay scheduled for expiry tracking"); + .expect("VOD streams should stay scheduled for expiry tracking"); - let refreshed_registration_ts = { + let unchanged_registration_ts = { let connections = manager.connections.read().await; connections.key_by_addr.get(&addr).expect("registration should still exist").ts }; - assert!(refreshed_registration_ts > previous_registration_ts); + assert_eq!(unchanged_registration_ts, previous_registration_ts); assert_eq!( deadline, - refreshed_registration_ts.saturating_add(manager.active_socket_ttl_secs()), - "continuous VOD streams should refresh their socket expiry deadline" + previous_registration_ts.saturating_add(manager.active_socket_ttl_secs()), + "deadline checks must not refresh VOD sockets without real body activity" ); } + #[tokio::test] + async fn touch_socket_activity_refreshes_registration_without_resetting_stream_start() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-vod-touch".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-vod-touch".to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + let mut channel = test_channel(8889); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = "http://localhost/movie-2.mkv".intern(); + + let stream = manager + .update_connection(ActiveUserConnectionParams { + uid: 603, + meter_uid: 703, + username: "user-vod-touch", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await + .expect("vod stream should be created"); + + let stale_registration_ts = { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); + registration.ts + }; + + manager.touch_socket_activity(&addr).await; + + let (refreshed_registration_ts, stream_started_at) = { + let connections = manager.connections.read().await; + let registration_ts = connections.key_by_addr.get(&addr).expect("registration should still exist").ts; + let stream_started_at = connections + .by_key + .get("user-vod-touch") + .and_then(|data| data.streams.iter().find(|active| active.uid == stream.uid)) + .expect("stream should still exist") + .ts; + (registration_ts, stream_started_at) + }; + + assert!(refreshed_registration_ts > stale_registration_ts); + assert_eq!(stream_started_at, stream.ts, "body activity must not reset visible stream duration"); + } + #[tokio::test] async fn update_session_addr_prunes_previous_registration_for_socket_bound_session() { let config = Config::default(); @@ -7474,6 +7677,228 @@ mod tests { ); } + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn origin_policy_refresh_promotes_counted_soft_session_when_hard_slot_is_available() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-hls-policy-refresh"); + user.max_connections = 1; + user.soft_connections = 1; + + let normal_addr: SocketAddr = "127.0.0.1:55185".parse().unwrap(); + let soft_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap(); + let normal_fingerprint = Fingerprint::new("fp-hls-policy-1".to_string(), "127.0.0.1".to_string(), normal_addr); + let soft_fingerprint = Fingerprint::new("fp-hls-policy-2".to_string(), "127.0.0.1".to_string(), soft_addr); + + manager.add_connection(&normal_addr).await; + manager.add_connection(&soft_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-normal", + virtual_id: 9210, + provider: "provider-a", + stream_url: "http://localhost/live-normal.m3u8", + addr: &normal_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-soft", + virtual_id: 9211, + provider: "provider-a", + stream_url: "http://localhost/live-soft.m3u8", + addr: &soft_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let normal_admission = manager + .connection_admission_for_session_activation(&user.username, user.max_connections, user.soft_connections, "tok-normal") + .await; + assert_eq!(normal_admission.permission, UserConnectionPermission::Allowed); + assert_eq!(normal_admission.kind, Some(ConnectionKind::Normal)); + manager + .update_connection(ActiveUserConnectionParams { + uid: 411, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &normal_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9210), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-normal"), + }) + .await + .expect("normal stream should bind"); + + let soft_admission = manager + .connection_admission_for_session_activation(&user.username, user.max_connections, user.soft_connections, "tok-soft") + .await; + assert_eq!(soft_admission.permission, UserConnectionPermission::Allowed); + assert_eq!(soft_admission.kind, Some(ConnectionKind::Soft)); + manager + .update_connection(ActiveUserConnectionParams { + uid: 412, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Soft, + priority: 0, + soft_priority: 0, + fingerprint: &soft_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9211), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-soft"), + }) + .await + .expect("soft stream should bind"); + + assert!(manager + .release_session_streams_and_counted_reservation(&user.username, "tok-normal") + .await); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.counts.normal, 0); + assert_eq!(connection_data.counts.soft, 1); + assert_eq!( + connection_data + .sessions + .iter() + .find(|session| session.token == "tok-soft") + .and_then(|session| session.connection_kind), + Some(ConnectionKind::Soft) + ); + } + + let refreshed_kind = manager + .refresh_session_connection_kind_for_origin_policy( + &user.username, + user.max_connections, + user.soft_connections, + "tok-soft", + ) + .await; + assert_eq!(refreshed_kind, Some(ConnectionKind::Normal)); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.counts.normal, 1); + assert_eq!(connection_data.counts.soft, 0); + assert_eq!( + connection_data + .sessions + .iter() + .find(|session| session.token == "tok-soft") + .and_then(|session| session.connection_kind), + Some(ConnectionKind::Normal) + ); + assert_eq!(connection_data.stream_kinds.get(&412), Some(&ConnectionKind::Normal)); + } + + #[tokio::test] + async fn origin_policy_refresh_returns_none_for_pending_grace_without_available_slot() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-pending-grace-origin-policy"); + user.max_connections = 1; + + let active_addr: SocketAddr = "127.0.0.1:55195".parse().unwrap(); + let pending_addr: SocketAddr = "127.0.0.1:55196".parse().unwrap(); + let active_fingerprint = Fingerprint::new("active".to_string(), "127.0.0.1".to_string(), active_addr); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-active", + virtual_id: 9301, + provider: "provider-a", + stream_url: "http://localhost/live-active.m3u8", + addr: &active_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 9301, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &active_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9301), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-active"), + }) + .await + .expect("active stream should bind the only normal slot"); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending", + virtual_id: 9302, + provider: "provider-a", + stream_url: "http://localhost/live-pending.m3u8", + addr: &pending_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: None, + socket_bound: false, + }) + .await; + manager + .mark_pending_provider( + &user.username, + "tok-pending", + PendingProviderReason::GraceHold, + current_time_secs() + 30, + ) + .await + .expect("pending session should be marked"); + + let refreshed_kind = manager + .refresh_session_connection_kind_for_origin_policy( + &user.username, + user.max_connections, + user.soft_connections, + "tok-pending", + ) + .await; + + assert_eq!(refreshed_kind, None); + } + #[tokio::test] async fn release_unbound_session_reservation_frees_reserved_slot() { let config = Config::default(); diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index f39ea2538..e05315e45 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -8,8 +8,8 @@ use crate::{ model::{ metadata_update_manager::MetadataUpdateManager, qos_aggregation_manager::exec_qos_aggregation, ActiveProviderManager, ActiveUserManager, - ConnectionManager, DownloadQueue, EventManager, PlaylistStorage, PlaylistStorageState, SharedStreamManager, - UpdateGuard, + ConnectionManager, DownloadQueue, EventManager, HlsProvisioningState, HlsProxyManager, PlaylistStorage, + PlaylistStorageState, SharedStreamManager, UpdateGuard, }, scheduler::exec_scheduler, }, @@ -187,6 +187,7 @@ fn cancel_services(app_state: &Arc, changes: &UpdateChanges) { metadata, qos_aggregation, downloads, + hls_cache: cancel_tokens.hls_cache.clone(), }; app_state.cancel_tokens.store(Arc::new(tokens)); @@ -361,6 +362,7 @@ pub struct CancelTokens { pub(crate) metadata: CancellationToken, pub(crate) qos_aggregation: CancellationToken, pub(crate) downloads: CancellationToken, + pub(crate) hls_cache: CancellationToken, } impl Default for CancelTokens { fn default() -> Self { @@ -372,6 +374,7 @@ impl Default for CancelTokens { metadata: CancellationToken::new(), qos_aggregation: CancellationToken::new(), downloads: CancellationToken::new(), + hls_cache: CancellationToken::new(), } } } @@ -404,6 +407,8 @@ pub struct AppState { pub downloads: Arc, pub cache: Arc>>, pub shared_stream_manager: Arc, + pub hls_proxy: Arc, + pub hls_provisioning: Arc, pub active_users: Arc, pub active_provider: Arc, pub connection_manager: Arc, @@ -434,6 +439,7 @@ impl AppState { self.app_config.set_config(config)?; reload_logger(config_log_level.as_deref()); self.active_provider.update_config(&self.app_config).await; + self.hls_proxy.update_config(&self.app_config).await; self.update_config().await?; let geoip_reload_needed = diff --git a/backend/src/api/model/connection_manager.rs b/backend/src/api/model/connection_manager.rs index f8228bdf3..b5008ebe5 100644 --- a/backend/src/api/model/connection_manager.rs +++ b/backend/src/api/model/connection_manager.rs @@ -1,7 +1,7 @@ use crate::{ api::model::{ ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, CustomVideoStreamType, EventManager, - EventMessage, ProviderHandle, SharedStreamManager, + EventMessage, ProviderHandle, SharedStreamManager, uses_direct_body_idle_timeout, }, model::StreamHistoryConfig, auth::Fingerprint, @@ -487,7 +487,14 @@ async fn release_stream_with_disconnect( } else { deps.user_manager.release_stream(&addr).await }; - let stream_info = stream_info?; + let Some(stream_info) = stream_info else { + debug_if_enabled!( + "Stream release skipped: no active stream for {} uid={:?}", + sanitize_sensitive_info(&addr.to_string()), + stream_uid + ); + return None; + }; let (bytes_sent, first_byte_latency_ms) = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; deps.event_manager.unregister_meter_client(stream_info.uid).await; let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); @@ -514,12 +521,15 @@ enum SocketActivityEvent { HttpActivity { addr: SocketAddr, }, + DirectBodyActivity { + addr: SocketAddr, + }, } impl SocketActivityEvent { fn addr(&self) -> SocketAddr { match self { - Self::HttpActivity { addr, .. } => *addr, + Self::HttpActivity { addr, .. } | Self::DirectBodyActivity { addr, .. } => *addr, } } } @@ -557,6 +567,12 @@ pub struct ConnectionParams<'a> { pub session_token: Option<&'a str>, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConnectionHistoryMode { + EmitConnect, + RefreshOnly, +} + impl ConnectionManager { pub fn new( user_manager: &Arc, @@ -676,7 +692,10 @@ impl ConnectionManager { expiry_index: &mut HashMap, user_manager: &Arc, ) { - let SocketActivityEvent::HttpActivity { addr } = event; + let addr = event.addr(); + if matches!(event, SocketActivityEvent::DirectBodyActivity { .. }) { + user_manager.touch_socket_activity(&addr).await; + } if let Some(expires_at) = user_manager.socket_expiry_deadline(&addr).await { let current = expiry_index.insert(addr, expires_at); @@ -722,6 +741,10 @@ impl ConnectionManager { // Fallthrough to release connection if `None` or `< now` expiry_index.remove(&addr); + debug_if_enabled!( + "Socket activity deadline expired for {}, releasing connection", + sanitize_sensitive_info(&addr.to_string()) + ); if cleanup_tx.send(CleanupEvent::ReleaseConnection { addr }).await.is_err() { debug!("Cleanup channel closed, stopping socket expiry worker"); break; @@ -987,7 +1010,7 @@ impl ConnectionManager { pub fn next_stream_uid(&self) -> u32 { self.stream_uid_counter - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { let next = current.wrapping_add(1); Some(if next == 0 { 1 } else { next }) }) @@ -1047,10 +1070,25 @@ impl ConnectionManager { self.socket_activity_tracker.track(SocketActivityEvent::HttpActivity { addr: *addr }); } + pub(crate) fn touch_direct_body_activity(&self, addr: &SocketAddr) { + self.socket_activity_tracker + .track(SocketActivityEvent::DirectBodyActivity { addr: *addr }); + } + pub async fn update_connection(&self, update: ConnectionParams<'_>) -> Option { + self.update_connection_with_history_mode(update, ConnectionHistoryMode::EmitConnect) + .await + } + + pub async fn update_connection_with_history_mode( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + ) -> Option { let uid = self.next_stream_uid(); let username = update.username; let fingerprint = update.fingerprint; + let track_direct_body_activity = uses_direct_body_idle_timeout(update.stream_channel); if let Some(stream_info) = self .user_manager .update_connection(ActiveUserConnectionParams { @@ -1073,7 +1111,16 @@ impl ConnectionManager { self.event_manager .register_meter_client(stream_info.uid, stream_info.meter_uid) .await; - emit_connect_record(&self.history_writer, &stream_info); + if history_mode == ConnectionHistoryMode::EmitConnect { + emit_connect_record(&self.history_writer, &stream_info); + } + if track_direct_body_activity { + debug_if_enabled!( + "Direct body stream registered for socket expiry: {}", + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + self.touch_direct_body_activity(&fingerprint.addr); + } self.event_manager .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); Some(stream_info) @@ -1366,7 +1413,7 @@ mod tests { tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); - tracker.track(SocketActivityEvent::HttpActivity { addr: addr_two }); + tracker.track(SocketActivityEvent::DirectBodyActivity { addr: addr_two }); let pending = tracker.drain(); assert_eq!(pending.len(), 2); @@ -1376,7 +1423,7 @@ mod tests { ))); assert!(pending.iter().any(|event| matches!( event, - SocketActivityEvent::HttpActivity { addr } if *addr == addr_two + SocketActivityEvent::DirectBodyActivity { addr } if *addr == addr_two ))); } diff --git a/backend/src/api/model/event_manager.rs b/backend/src/api/model/event_manager.rs index 4df0ef180..f1ad5e24b 100644 --- a/backend/src/api/model/event_manager.rs +++ b/backend/src/api/model/event_manager.rs @@ -141,7 +141,7 @@ impl EventManager { pub fn stream_meter_subscriber_disconnected(&self) { let _ = self .stream_meter_subscriber_count - .fetch_update(Ordering::AcqRel, Ordering::Relaxed, |count| count.checked_sub(1)); + .try_update(Ordering::AcqRel, Ordering::Relaxed, |count| count.checked_sub(1)); } pub fn has_stream_meter_subscribers(&self) -> bool { diff --git a/backend/src/api/model/hls_cache/backpressure.rs b/backend/src/api/model/hls_cache/backpressure.rs new file mode 100644 index 000000000..c764d3b5f --- /dev/null +++ b/backend/src/api/model/hls_cache/backpressure.rs @@ -0,0 +1,59 @@ +use super::HlsSession; + +/// Coarse pressure level for scheduling live HLS origin work. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsBackpressureState { + Normal, + Degraded, + Saturated, +} + +impl HlsBackpressureState { + pub const fn allows_prefetch(self) -> bool { matches!(self, Self::Normal) } + + pub const fn allows_new_demand_fetch(self) -> bool { !matches!(self, Self::Saturated) } +} + +pub fn classify_hls_backpressure( + session: &HlsSession, + global_available_permits: usize, + max_session_segment_fetches: usize, +) -> HlsBackpressureState { + if global_available_permits == 0 || session.active_segment_fetches >= max_session_segment_fetches { + return HlsBackpressureState::Saturated; + } + if global_available_permits == 1 || session.active_segment_fetches > 0 { + return HlsBackpressureState::Degraded; + } + HlsBackpressureState::Normal +} + +#[cfg(test)] +mod tests { + use super::{classify_hls_backpressure, HlsBackpressureState}; + use crate::api::model::{HlsSession, HlsSessionKey}; + + fn session() -> HlsSession { HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0) } + + #[test] + fn classifies_saturated_when_global_slots_are_exhausted() { + let session = session(); + + assert_eq!(classify_hls_backpressure(&session, 0, 2), HlsBackpressureState::Saturated); + } + + #[test] + fn classifies_saturated_when_session_slots_are_exhausted() { + let mut session = session(); + session.active_segment_fetches = 2; + + assert_eq!(classify_hls_backpressure(&session, 3, 2), HlsBackpressureState::Saturated); + } + + #[test] + fn classifies_degraded_when_capacity_is_low() { + let session = session(); + + assert_eq!(classify_hls_backpressure(&session, 1, 2), HlsBackpressureState::Degraded); + } +} diff --git a/backend/src/api/model/hls_cache/cache.rs b/backend/src/api/model/hls_cache/cache.rs new file mode 100644 index 000000000..26791852d --- /dev/null +++ b/backend/src/api/model/hls_cache/cache.rs @@ -0,0 +1,999 @@ +use super::{ProxyMapId, ProxySessionId, TransientResourceId}; +use std::{ + collections::{HashMap, HashSet}, + fmt, io, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, RwLock as StdRwLock, + }, + time::{Duration, SystemTime}, +}; +use tokio::{ + fs::{self, File, OpenOptions}, + io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}, + sync::{Mutex, RwLock}, + time::timeout, +}; + +pub const DEFAULT_HLS_CACHE_PATH: &str = "/tmp/tuliprox/cache/hls"; +pub const DEFAULT_HLS_CACHE_DURATION_SECS: u64 = 300; +const TEMP_CREATE_ATTEMPTS: usize = 8; + +/// Stable cache key for one proxy-visible HLS segment. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct SegmentCacheKey { + session_id: ProxySessionId, + seq: u64, + file_ext: String, +} + +impl SegmentCacheKey { + pub fn new(proxy_session_id: ProxySessionId, proxy_seq: u64, proxy_file_ext: impl Into) -> Self { + Self { session_id: proxy_session_id, seq: proxy_seq, file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls:{}:{:020}", self.session_id.0, self.seq) } + + pub fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } + + pub fn proxy_seq(&self) -> u64 { self.seq } +} + +impl fmt::Debug for SegmentCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentCacheKey") + .field("session_id", &"") + .field("seq", &self.seq) + .field("file_ext", &self.file_ext) + .finish() + } +} + +/// Stable cache key for one proxy-visible HLS EXT-X-MAP object. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct MapCacheKey { + session_id: ProxySessionId, + map_id: ProxyMapId, + file_ext: String, +} + +impl MapCacheKey { + pub fn new( + proxy_session_id: ProxySessionId, + proxy_map_id: impl Into, + proxy_file_ext: impl Into, + ) -> Self { + Self { session_id: proxy_session_id, map_id: proxy_map_id.into(), file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls-map:{}:{:020}", self.session_id.0, self.map_id.0) } + + pub fn proxy_map_id(&self) -> ProxyMapId { self.map_id } +} + +impl fmt::Debug for MapCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapCacheKey") + .field("session_id", &"") + .field("map_id", &self.map_id.0) + .field("file_ext", &self.file_ext) + .finish() + } +} + +/// Stable cache key for one demand-cached transient passthrough object. +/// +/// This key is not a provider-source identity. It keys the proxy-visible object for a concrete transient resource. +/// The concrete origin fetch URI is kept in `TransientResourceRef`; callers must not reconstruct it from this key or +/// force host-neutral cache hits across redirect/CDN contexts without a separate safe resource identity. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct TransientObjectCacheKey { + session_id: ProxySessionId, + resource_id: TransientResourceId, + file_ext: String, +} + +impl TransientObjectCacheKey { + pub fn new( + proxy_session_id: ProxySessionId, + transient_resource_id: TransientResourceId, + proxy_file_ext: impl Into, + ) -> Self { + Self { session_id: proxy_session_id, resource_id: transient_resource_id, file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls-transient:{}:{}", self.session_id.0, self.resource_id.0) } + + pub fn transient_resource_id(&self) -> &TransientResourceId { &self.resource_id } +} + +impl fmt::Debug for TransientObjectCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientObjectCacheKey") + .field("session_id", &"") + .field("resource_id", &self.resource_id) + .field("file_ext", &self.file_ext) + .finish() + } +} + +/// Resolves a proxy cache object to a safe path below the configured HLS cache root. +pub trait HlsCacheObjectKey { + fn session_path_component(&self) -> String; + fn file_name(&self) -> String; +} + +impl HlsCacheObjectKey for SegmentCacheKey { + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("{:06}.{}", self.seq, self.file_ext) } +} + +impl HlsCacheObjectKey for MapCacheKey { + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("map/{:06}.{}", self.map_id.0, self.file_ext) } +} + +impl HlsCacheObjectKey for TransientObjectCacheKey { + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("r/{}.{}", self.resource_id.0, self.file_ext) } +} + +/// Filesystem metadata for a committed HLS cache object. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CachedSegmentMetadata { + pub path: PathBuf, + pub size: u64, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct StagedCacheObject { + pub path: PathBuf, + pub size: u64, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum CacheInvalidationOutcome { + Invalidated, + DeferredActiveTempFiles, +} + +/// File-backed cache for committed HLS segment objects. +pub struct HlsSegmentCache { + cache_path: StdRwLock, + active_temp_files: Arc>>, + max_object_bytes: AtomicU64, + max_cache_bytes: AtomicU64, + max_session_bytes: AtomicU64, + marker_path: StdRwLock>, + // serialize commits for exact budgets; shard accounting only if measured commit contention requires it. + capacity: Mutex, +} + +#[derive(Default)] +struct CacheCapacityState { + cache_path: PathBuf, + initialized: bool, + total_bytes: u64, + session_bytes: HashMap, +} + +impl HlsSegmentCache { + pub fn new() -> Self { Self::with_cache_path(DEFAULT_HLS_CACHE_PATH) } + + pub fn with_cache_path(cache_path: impl Into) -> Self { + Self { + cache_path: StdRwLock::new(cache_path.into()), + active_temp_files: Arc::new(RwLock::new(HashSet::new())), + max_object_bytes: AtomicU64::new(u64::MAX), + max_cache_bytes: AtomicU64::new(u64::MAX), + max_session_bytes: AtomicU64::new(u64::MAX), + marker_path: StdRwLock::new(None), + capacity: Mutex::new(CacheCapacityState::default()), + } + } + + pub fn cache_path(&self) -> PathBuf { self.cache_path_snapshot() } + + pub fn update_cache_path(&self, cache_path: impl Into) -> bool { + let cache_path = cache_path.into(); + let mut current = self.cache_path.write().unwrap_or_else(std::sync::PoisonError::into_inner); + if *current == cache_path { + return false; + } + *current = cache_path; + *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + true + } + + pub fn update_cache_limits(&self, max_cache_bytes: u64, max_session_bytes: u64) { + let max_cache_bytes = max_cache_bytes.max(1); + let max_session_bytes = max_session_bytes.max(1); + self.max_cache_bytes.store(max_cache_bytes, Ordering::Release); + self.max_session_bytes.store(max_session_bytes, Ordering::Release); + self.max_object_bytes.store(max_cache_bytes.min(max_session_bytes), Ordering::Release); + } + + pub async fn metadata(&self, key: &K) -> io::Result> { + let path = self.path_for_key(key); + match fs::metadata(&path).await { + Ok(metadata) => Ok(Some(CachedSegmentMetadata { path, size: metadata.len() })), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + pub async fn open_range(&self, key: &K, start: u64) -> io::Result { + let mut file = File::open(self.path_for_key(key)).await?; + file.seek(SeekFrom::Start(start)).await?; + Ok(file) + } + + pub async fn write_temp_and_commit(&self, key: &K, mut reader: R) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.write_temp_and_commit_inner(key, &mut reader, None).await + } + + pub async fn write_temp_and_commit_with_timeout( + &self, + key: &K, + mut reader: R, + deadline: Duration, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.write_temp_and_commit_inner(key, &mut reader, Some(deadline)).await + } + + pub async fn stage_temp_with_timeout( + &self, + key: &K, + mut reader: R, + deadline: Duration, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner(key, &mut reader, Some(deadline)).await + } + + pub async fn commit_staged(&self, key: &K, staged: StagedCacheObject) -> io::Result + where + K: HlsCacheObjectKey, + { + let result = self.commit_staged_inner(key, &staged).await; + if result.is_err() { + let _ = fs::remove_file(&staged.path).await; + } + self.unregister_temp_file(&staged.path).await; + result + } + + async fn commit_staged_inner(&self, key: &K, staged: &StagedCacheObject) -> io::Result + where + K: HlsCacheObjectKey, + { + self.ensure_cache_root_marker().await?; + if staged.size > self.max_object_bytes.load(Ordering::Acquire) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "hls cache object exceeds configured size limit")); + } + let final_path = self.path_for_key(key); + let Some(parent) = final_path.parent() else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); + }; + fs::create_dir_all(parent).await?; + let session_component = key.session_path_component(); + let cache_path = self.cache_path_snapshot(); + let mut capacity = self.capacity.lock().await; + if !capacity.initialized || capacity.cache_path != cache_path { + let (total_bytes, session_bytes) = scan_committed_cache_usage(&cache_path).await?; + *capacity = CacheCapacityState { + cache_path: cache_path.clone(), + initialized: true, + total_bytes, + session_bytes, + }; + } + let old_size = match fs::metadata(&final_path).await { + Ok(metadata) => metadata.len(), + Err(err) if err.kind() == io::ErrorKind::NotFound => 0, + Err(err) => return Err(err), + }; + let session_size = capacity.session_bytes.get(&session_component).copied().unwrap_or_default(); + let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(staged.size); + let projected_session = session_size.saturating_sub(old_size).saturating_add(staged.size); + if projected_total > self.max_cache_bytes.load(Ordering::Acquire) + || projected_session > self.max_session_bytes.load(Ordering::Acquire) + { + return Err(io::Error::other("hls cache capacity exceeded")); + } + if self.cache_path_snapshot() != cache_path { + return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during object write")); + } + fs::rename(&staged.path, &final_path).await?; + capacity.total_bytes = projected_total; + capacity.session_bytes.insert(session_component, projected_session); + drop(capacity); + self.metadata(key).await?.ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "committed segment cache file is missing after atomic rename") + }) + } + + pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> { + let result = match fs::remove_file(&staged.path).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + }; + self.unregister_temp_file(&staged.path).await; + result + } + + async fn write_temp_and_commit_inner( + &self, + key: &K, + reader: &mut R, + deadline: Option, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let staged = self.stage_temp_inner(key, reader, deadline).await?; + self.commit_staged(key, staged).await + } + + async fn stage_temp_inner( + &self, + key: &K, + reader: &mut R, + deadline: Option, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.ensure_cache_root_marker().await?; + let final_path = self.path_for_key(key); + let Some(parent) = final_path.parent() else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); + }; + fs::create_dir_all(parent).await?; + + let (temp_path, mut temp_file) = self.create_temp_file(key).await?; + let copy = async { + let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); + let mut limited = reader.take(max_object_bytes.saturating_add(1)); + let size = tokio::io::copy(&mut limited, &mut temp_file).await?; + if size > max_object_bytes { + return Err(io::Error::new(io::ErrorKind::InvalidData, "hls cache object exceeds configured size limit")); + } + temp_file.flush().await?; + drop(temp_file); + Ok::(size) + }; + let copy_result = if let Some(deadline) = deadline { + if let Ok(result) = timeout(deadline, copy).await { + result + } else { + let _ = fs::remove_file(&temp_path).await; + self.unregister_temp_file(&temp_path).await; + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } + } else { + copy.await + }; + match copy_result { + Ok(size) => Ok(StagedCacheObject { path: temp_path, size }), + Err(err) => { + let _ = fs::remove_file(&temp_path).await; + self.unregister_temp_file(&temp_path).await; + Err(err) + } + } + } + + pub async fn write_bytes_and_commit(&self, key: &K, bytes: &[u8]) -> io::Result + where + K: HlsCacheObjectKey, + { + self.write_temp_and_commit(key, bytes).await + } + + pub async fn delete(&self, key: &K) -> io::Result<()> { + let path = self.path_for_key(key); + let mut capacity = self.capacity.lock().await; + let size = fs::metadata(&path).await.map_or(0, |metadata| metadata.len()); + let result = match fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + }; + if result.is_ok() + && size > 0 + && capacity.initialized + && capacity.cache_path == self.cache_path_snapshot() + { + capacity.total_bytes = capacity.total_bytes.saturating_sub(size); + let session_component = key.session_path_component(); + if let Some(session_bytes) = capacity.session_bytes.get_mut(&session_component) { + *session_bytes = session_bytes.saturating_sub(size); + } + } + result + } + + pub fn object_path(&self, key: &K) -> PathBuf { self.path_for_key(key) } + + pub async fn delete_temp_files_older_than(&self, cutoff: SystemTime) -> io::Result { + let cache_path = self.cache_path_snapshot(); + let active_temp_files = self.active_temp_files.read().await.clone(); + delete_temp_files_older_than(&cache_path, cutoff, &active_temp_files).await + } + + pub async fn delete_session_dir(&self, proxy_session_id: &ProxySessionId) -> io::Result<()> { + let path = self.cache_path_snapshot().join(safe_session_path_component(proxy_session_id)); + let mut capacity = self.capacity.lock().await; + let result = match fs::remove_dir_all(path).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + }; + if result.is_ok() && capacity.initialized && capacity.cache_path == self.cache_path_snapshot() { + let session_component = safe_session_path_component(proxy_session_id); + if let Some(bytes) = capacity.session_bytes.remove(&session_component) { + capacity.total_bytes = capacity.total_bytes.saturating_sub(bytes); + } + } + result + } + + pub async fn delete_orphan_session_dirs( + &self, + active_session_ids: &HashSet, + freshness_cutoff: SystemTime, + ) -> io::Result { + let cache_path = self.cache_path_snapshot(); + ensure_safe_cache_root(&cache_path).await?; + let active_paths = active_session_ids + .iter() + .map(|id| cache_path.join(safe_session_path_component(id))) + .collect::>(); + let active_temp_files = self.active_temp_files.write().await; + let mut entries = fs::read_dir(&cache_path).await?; + let mut removed = 0_usize; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if !entry.file_type().await?.is_dir() + || active_paths.contains(&path) + || active_temp_files.iter().any(|temp| temp.starts_with(&path)) + { + continue; + } + // Freshness guard: skip directories committed after the GC took its + // in-memory session snapshot. Their owning session may not yet be + // visible in `active_session_ids`, so deleting them would race with + // a concurrent segment write that just created the directory. + let metadata = match entry.metadata().await { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + if let Ok(modified) = metadata.modified() { + if modified > freshness_cutoff { + continue; + } + } + match fs::remove_dir_all(&path).await { + Ok(()) => removed = removed.saturating_add(1), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + drop(active_temp_files); + if removed > 0 { + self.capacity.lock().await.initialized = false; + } + Ok(removed) + } + + pub async fn has_active_temp_files_for_session(&self, proxy_session_id: &ProxySessionId) -> bool { + let session_path = self.cache_path_snapshot().join(safe_session_path_component(proxy_session_id)); + self.active_temp_files.read().await.iter().any(|path| path.starts_with(&session_path)) + } + + pub async fn has_active_temp_files(&self) -> bool { !self.active_temp_files.read().await.is_empty() } + + pub async fn invalidate_all_if_no_active_temp_files(&self) -> io::Result { + let active_temp_files = self.active_temp_files.write().await; + if !active_temp_files.is_empty() { + return Ok(CacheInvalidationOutcome::DeferredActiveTempFiles); + } + self.invalidate_all_unchecked().await?; + Ok(CacheInvalidationOutcome::Invalidated) + } + + pub async fn invalidate_all(&self) -> io::Result<()> { self.invalidate_all_unchecked().await } + + async fn invalidate_all_unchecked(&self) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + ensure_safe_cache_root(&cache_path).await?; + fs::create_dir_all(&cache_path).await?; + let mut entries = fs::read_dir(&cache_path).await?; + while let Some(entry) = entries.next_entry().await? { + if entry.file_name() == REWRITE_SECRET_FINGERPRINT_FILE || entry.file_name() == HLS_CACHE_ROOT_MARKER_FILE { + continue; + } + let file_type = entry.file_type().await?; + if file_type.is_dir() { + fs::remove_dir_all(entry.path()).await?; + } else { + fs::remove_file(entry.path()).await?; + } + } + self.capacity.lock().await.initialized = false; + Ok(()) + } + + pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result> { + match fs::read_to_string(self.rewrite_secret_fingerprint_path()).await { + Ok(value) => Ok(Some(value.trim().to_string())), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + pub async fn write_rewrite_secret_fingerprint(&self, fingerprint: &str) -> io::Result<()> { + self.ensure_cache_root_marker().await?; + fs::write(self.rewrite_secret_fingerprint_path(), fingerprint).await + } + + fn path_for_key(&self, key: &K) -> PathBuf { + self.cache_path_snapshot().join(key.session_path_component()).join(key.file_name()) + } + + async fn create_temp_file(&self, key: &K) -> io::Result<(PathBuf, File)> { + let mut active_temp_files = self.active_temp_files.write().await; + for _ in 0..TEMP_CREATE_ATTEMPTS { + let temp_path = self.temp_path_for_key(key); + match OpenOptions::new().write(true).create_new(true).open(&temp_path).await { + Ok(file) => { + active_temp_files.insert(temp_path.clone()); + return Ok((temp_path, file)); + } + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {} + Err(err) => return Err(err), + } + } + Err(io::Error::new(io::ErrorKind::AlreadyExists, "could not create unique hls cache temp file")) + } + + fn temp_path_for_key(&self, key: &K) -> PathBuf { + let suffix = fastrand::u64(..); + self.cache_path_snapshot() + .join(key.session_path_component()) + .join(format!("{}.tmp.{suffix:016x}", key.file_name())) + } + + async fn unregister_temp_file(&self, temp_path: &Path) { self.active_temp_files.write().await.remove(temp_path); } + + fn rewrite_secret_fingerprint_path(&self) -> PathBuf { + self.cache_path_snapshot().join(REWRITE_SECRET_FINGERPRINT_FILE) + } + + async fn ensure_cache_root_marker(&self) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + if self + .marker_path + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + == Some(&cache_path) + { + return Ok(()); + } + ensure_not_root_like_cache_path(&cache_path)?; + fs::create_dir_all(&cache_path).await?; + fs::write(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE), b"tuliprox-hls-cache\n").await?; + *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cache_path); + Ok(()) + } + + fn cache_path_snapshot(&self) -> PathBuf { + self.cache_path.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + } +} + +const REWRITE_SECRET_FINGERPRINT_FILE: &str = ".rewrite_secret_fingerprint"; +const HLS_CACHE_ROOT_MARKER_FILE: &str = ".tuliprox-hls-cache-root"; + +fn safe_session_path_component(proxy_session_id: &ProxySessionId) -> String { + let value = &proxy_session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() +} + +fn ensure_not_root_like_cache_path(cache_path: &Path) -> io::Result<()> { + if cache_path.as_os_str().is_empty() || cache_path.parent().is_none() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to invalidate unsafe hls cache root")); + } + Ok(()) +} + +async fn ensure_safe_cache_root(cache_path: &Path) -> io::Result<()> { + ensure_not_root_like_cache_path(cache_path)?; + match fs::metadata(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE)).await { + Ok(metadata) if metadata.is_file() => Ok(()), + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to invalidate hls cache root without marker file", + )), + Err(err) if err.kind() == io::ErrorKind::NotFound => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to invalidate hls cache root without marker file", + )), + Err(err) => Err(err), + } +} + +async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap)> { + let mut total_bytes = 0_u64; + let mut session_bytes = HashMap::new(); + let mut root_entries = match fs::read_dir(cache_path).await { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok((0, session_bytes)), + Err(err) => return Err(err), + }; + while let Some(root_entry) = root_entries.next_entry().await? { + if !root_entry.file_type().await?.is_dir() { + continue; + } + let session_component = root_entry.file_name().to_string_lossy().into_owned(); + let mut bytes = 0_u64; + let mut pending_dirs = vec![root_entry.path()]; + while let Some(dir) = pending_dirs.pop() { + let mut entries = fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { + let file_type = entry.file_type().await?; + if file_type.is_dir() { + pending_dirs.push(entry.path()); + } else if file_type.is_file() && !is_temp_cache_file(&entry.path()) { + bytes = bytes.saturating_add(entry.metadata().await?.len()); + } + } + } + total_bytes = total_bytes.saturating_add(bytes); + session_bytes.insert(session_component, bytes); + } + Ok((total_bytes, session_bytes)) +} + +async fn delete_temp_files_older_than( + root: &Path, + cutoff: SystemTime, + active_temp_files: &HashSet, +) -> io::Result { + let mut deleted = 0_usize; + let mut pending_dirs = vec![root.to_path_buf()]; + while let Some(dir) = pending_dirs.pop() { + let mut entries = match fs::read_dir(&dir).await { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let file_type = entry.file_type().await?; + if file_type.is_dir() { + pending_dirs.push(path); + continue; + } + if !is_temp_cache_file(&path) || active_temp_files.contains(&path) { + continue; + } + let metadata = entry.metadata().await?; + if metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH) >= cutoff { + continue; + } + match fs::remove_file(&path).await { + Ok(()) => deleted = deleted.saturating_add(1), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + } + Ok(deleted) +} + +fn is_temp_cache_file(path: &Path) -> bool { + path.file_name().and_then(|file_name| file_name.to_str()).is_some_and(|file_name| file_name.contains(".tmp.")) +} + +impl Default for HlsSegmentCache { + fn default() -> Self { Self::new() } +} + +#[cfg(test)] +mod tests { + use super::{CacheInvalidationOutcome, HlsSegmentCache, MapCacheKey, SegmentCacheKey, TransientObjectCacheKey}; + use crate::api::model::{build_transient_resource_id, ProxySessionId}; + use std::{collections::HashSet, io, sync::Arc, time::{Duration, SystemTime}}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn cache_key() -> SegmentCacheKey { SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 123, "ts") } + + #[test] + fn segment_cache_key_contains_no_origin_data() { + let key = cache_key(); + + assert_eq!(key.stable_value(), "hls:proxy_session:00000000000000000123"); + } + + #[test] + fn cache_key_debug_redacts_proxy_session_id() { + let key = SegmentCacheKey::new(ProxySessionId("secretToken".to_string()), 123, "ts"); + let debug = format!("{key:?}"); + + assert!(debug.contains("")); + assert!(!debug.contains("secretToken")); + assert!(!debug.contains(&key.stable_value())); + } + + #[test] + fn transient_object_cache_key_keeps_redirect_hosts_distinct_without_leaking_urls() { + let proxy_session_id = ProxySessionId("proxy_session".to_string()); + let first_resource = + build_transient_resource_id("https://cdn-a.example.net/live/redirected/seg001.ts", b"secret"); + let second_resource = + build_transient_resource_id("https://cdn-b.example.net/live/redirected/seg001.ts", b"secret"); + + let first = TransientObjectCacheKey::new(proxy_session_id.clone(), first_resource, "ts"); + let second = TransientObjectCacheKey::new(proxy_session_id, second_resource, "ts"); + + assert_ne!(first, second); + assert_ne!(first.stable_value(), second.stable_value()); + for value in [first.stable_value(), second.stable_value()] { + assert!(!value.contains("provider://")); + assert!(!value.contains("cdn-a.example.net")); + assert!(!value.contains("cdn-b.example.net")); + assert!(!value.contains("/live/redirected/seg001.ts")); + } + } + + #[tokio::test] + async fn write_temp_and_commit_creates_final_segment_cache_file_with_proxy_layout() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + + let metadata = cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + assert_eq!(metadata.size, 12); + assert!(metadata.path.exists()); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata.clone())); + assert!(metadata.path.ends_with("proxy_session/000123.ts")); + } + + #[tokio::test] + async fn write_temp_and_commit_creates_final_map_cache_file_with_proxy_layout() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = MapCacheKey::new(ProxySessionId("proxy_session".to_string()), 0, "mp4"); + + let metadata = cache.write_bytes_and_commit(&key, b"map-body").await.expect("commit should succeed"); + + assert_eq!(metadata.size, 8); + assert!(metadata.path.ends_with("proxy_session/map/000000.mp4")); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata)); + } + + #[tokio::test] + async fn write_temp_and_commit_with_timeout_cleans_active_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + let (_writer, reader) = tokio::io::duplex(64); + + let result = cache.write_temp_and_commit_with_timeout(&key, reader, Duration::from_millis(1)).await; + + assert_eq!(result.expect_err("commit should time out").kind(), io::ErrorKind::TimedOut); + assert!(!cache.has_active_temp_files().await); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); + } + + #[tokio::test] + async fn cache_object_write_rejects_bytes_above_the_configured_limit() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.update_cache_limits(3, 3); + let key = SegmentCacheKey::new(ProxySessionId("session".to_string()), 1, "ts"); + + let result = cache.write_bytes_and_commit(&key, b"four").await; + + assert!(result.is_err()); + assert!(cache.metadata(&key).await.expect("metadata").is_none()); + } + + #[tokio::test] + async fn orphan_session_cleanup_preserves_active_session_directories() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); + let active = ProxySessionId("active".to_string()); + let orphan = ProxySessionId("orphan".to_string()); + tokio::fs::create_dir_all(temp_dir.path().join("active")).await.expect("active dir"); + tokio::fs::create_dir_all(temp_dir.path().join("orphan")).await.expect("orphan dir"); + let cutoff = SystemTime::now(); + + let removed = cache + .delete_orphan_session_dirs(&HashSet::from([active]), cutoff) + .await + .expect("cleanup"); + + assert_eq!(removed, 1); + assert!(temp_dir.path().join("active").exists()); + assert!(!temp_dir.path().join(orphan.0).exists()); + } + + #[tokio::test] + async fn orphan_session_cleanup_skips_directories_newer_than_freshness_cutoff() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); + let stale = ProxySessionId("stale".to_string()); + let fresh = ProxySessionId("fresh".to_string()); + tokio::fs::create_dir_all(temp_dir.path().join(stale.0.clone())).await.expect("stale dir"); + tokio::fs::create_dir_all(temp_dir.path().join(fresh.0.clone())).await.expect("fresh dir"); + + // Set the fresh dir's mtime to the future relative to the cutoff. + let cutoff = SystemTime::now(); + let future = cutoff + std::time::Duration::from_secs(60); + filetime::set_file_mtime(temp_dir.path().join(fresh.0.clone()), filetime::FileTime::from_system_time(future)) + .expect("set fresh mtime"); + + let removed = cache + .delete_orphan_session_dirs(&HashSet::new(), cutoff) + .await + .expect("cleanup"); + + assert_eq!(removed, 1, "stale orphan dir should be removed"); + assert!(!temp_dir.path().join(stale.0).exists()); + assert!( + temp_dir.path().join(fresh.0).exists(), + "directory newer than the freshness cutoff must be preserved to avoid racing concurrent session creation" + ); + } + + #[tokio::test] + async fn cache_commits_enforce_the_global_budget_across_sessions() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.update_cache_limits(5, 5); + let first = SegmentCacheKey::new(ProxySessionId("first".to_string()), 1, "ts"); + let second = SegmentCacheKey::new(ProxySessionId("second".to_string()), 1, "ts"); + + assert!(cache.write_bytes_and_commit(&first, b"123").await.is_ok()); + assert!(cache.write_bytes_and_commit(&second, b"456").await.is_err()); + assert!(cache.metadata(&second).await.expect("metadata").is_none()); + } + + #[tokio::test] + async fn open_range_reads_from_requested_offset() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + + let file = cache.open_range(&key, 4).await.expect("range should open"); + let mut body = Vec::new(); + file.take(3).read_to_end(&mut body).await.expect("range should read"); + + assert_eq!(body, b"456"); + } + + #[tokio::test] + async fn temp_file_collision_does_not_overwrite_existing_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + let parent = temp_dir.path().join("proxy_session"); + tokio::fs::create_dir_all(&parent).await.expect("parent should be created"); + let existing_temp = parent.join("000123.ts.tmp.0000000000000000"); + tokio::fs::write(&existing_temp, b"existing").await.expect("temp fixture should write"); + + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + assert_eq!(tokio::fs::read(&existing_temp).await.expect("existing temp should remain"), b"existing"); + } + + #[tokio::test] + async fn invalidate_all_if_no_active_temp_files_deletes_only_when_no_temp_write_is_active() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let committed_key = cache_key(); + cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("commit should succeed"); + + let outcome = cache.invalidate_all_if_no_active_temp_files().await.expect("invalidation should succeed"); + + assert_eq!(outcome, CacheInvalidationOutcome::Invalidated); + assert_eq!(cache.metadata(&committed_key).await.expect("metadata should read"), None); + + cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("second commit should succeed"); + let active_key = SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 124, "ts"); + let (mut writer, reader) = tokio::io::duplex(64); + let task_cache = Arc::clone(&cache); + let task_key = active_key.clone(); + let write_task = tokio::spawn(async move { task_cache.write_temp_and_commit(&task_key, reader).await }); + for _ in 0..50 { + if cache.has_active_temp_files().await { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(cache.has_active_temp_files().await); + + let outcome = + cache.invalidate_all_if_no_active_temp_files().await.expect("deferred invalidation should succeed"); + + assert_eq!(outcome, CacheInvalidationOutcome::DeferredActiveTempFiles); + assert!(cache.metadata(&committed_key).await.expect("metadata should read").is_some()); + writer.write_all(b"done").await.expect("write temp body"); + drop(writer); + write_task.await.expect("temp write task joins").expect("temp write commits"); + } + + #[tokio::test] + async fn invalidate_all_refuses_unmarked_cache_root() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let unrelated_file = temp_dir.path().join("unrelated"); + tokio::fs::write(&unrelated_file, b"keep").await.expect("fixture should write"); + + let err = cache.invalidate_all().await.expect_err("unmarked root should be refused"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert!(unrelated_file.exists()); + } + + #[tokio::test] + async fn delete_removes_committed_file_idempotently() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + cache.delete(&key).await.expect("delete should succeed"); + cache.delete(&key).await.expect("second delete should be idempotent"); + + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); + } +} diff --git a/backend/src/api/model/hls_cache/deadline.rs b/backend/src/api/model/hls_cache/deadline.rs new file mode 100644 index 000000000..bd8ce0c83 --- /dev/null +++ b/backend/src/api/model/hls_cache/deadline.rs @@ -0,0 +1,38 @@ +use std::{pin::Pin, time::Duration}; +use tokio::time::Sleep; + +const HLS_CLIENT_BODY_SEND_TIMEOUT_SECS: u64 = 90; + +pub fn hls_client_body_send_deadline() -> Duration { Duration::from_secs(HLS_CLIENT_BODY_SEND_TIMEOUT_SECS) } + +pub fn refresh_hls_client_body_send_deadline(mut deadline: Pin<&mut Sleep>) { + deadline.as_mut().reset(tokio::time::Instant::now() + hls_client_body_send_deadline()); +} + +pub fn hls_object_body_deadline(timeout_ms: u64) -> Duration { Duration::from_millis(timeout_ms.max(1)) } + +#[cfg(test)] +mod tests { + use super::{hls_client_body_send_deadline, hls_object_body_deadline, refresh_hls_client_body_send_deadline}; + use std::time::Duration; + + #[test] + fn object_body_deadline_uses_configured_segment_timeout() { + assert_eq!(hls_object_body_deadline(10_000), Duration::from_secs(10)); + } + + #[test] + fn client_body_send_deadline_is_fixed_to_ninety_seconds() { + assert_eq!(hls_client_body_send_deadline(), Duration::from_secs(90)); + } + + #[tokio::test] + async fn client_body_send_deadline_refreshes_after_progress() { + let mut deadline = Box::pin(tokio::time::sleep(Duration::ZERO)); + let expired_at = deadline.deadline(); + + refresh_hls_client_body_send_deadline(deadline.as_mut()); + + assert!(deadline.deadline() > expired_at); + } +} diff --git a/backend/src/api/model/hls_cache/gc.rs b/backend/src/api/model/hls_cache/gc.rs new file mode 100644 index 000000000..0e2bad834 --- /dev/null +++ b/backend/src/api/model/hls_cache/gc.rs @@ -0,0 +1,1494 @@ +use super::{ + renderer_candidate_window_proxy_seqs, safe_proxy_session_id, CacheInvalidationOutcome, HlsCacheMetrics, + HlsSegmentCache, HlsSession, HlsExpiredSessionReason, HlsSessionHandle, HlsSessionStore, MapCacheKey, + MapCacheStatus, ProxyMapId, + ProxySessionId, SegmentCacheKey, SegmentCacheStatus, TransientObjectCacheKey, +}; +use crate::{api::model::AppState, model::HlsCacheConfig}; +use arc_swap::ArcSwap; +use log::{debug, error, info, warn}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashSet, + io, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tokio_util::sync::CancellationToken; + +const HLS_CACHE_GC_INTERVAL: Duration = Duration::from_secs(30); +const DEFAULT_TEMP_FILE_RETENTION_MS: u64 = 30_000; +const DEFAULT_FAILED_SEGMENT_RETENTION_MS: u64 = 10_000; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct GarbageCollectionPolicy { + pub cache_duration_ms: u64, + pub cache_bytes_global: u64, + pub cache_bytes_per_session: u64, + pub session_idle_timeout_ms: u64, + pub temp_file_retention_ms: u64, + pub failed_segment_retention_ms: u64, +} + +impl GarbageCollectionPolicy { + pub fn from_config(config: &HlsCacheConfig) -> Self { + Self { + cache_duration_ms: config.cache_duration.saturating_mul(1_000), + cache_bytes_global: config.cache_bytes, + cache_bytes_per_session: config.cache_bytes_per_session, + session_idle_timeout_ms: config.session_idle_timeout.saturating_mul(1_000), + temp_file_retention_ms: DEFAULT_TEMP_FILE_RETENTION_MS, + failed_segment_retention_ms: DEFAULT_FAILED_SEGMENT_RETENTION_MS, + } + } +} + +impl Default for GarbageCollectionPolicy { + fn default() -> Self { + let default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); + Self::from_config(&default_config) + } +} + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct ProtectedSet { + pub segment_proxy_seqs: HashSet, + pub map_ids: HashSet, +} + +impl ProtectedSet { + pub fn from_session(session: &HlsSession) -> Self { + let mut protected = Self::default(); + + if let Some(rendered) = &session.last_rendered_manifest { + protected.segment_proxy_seqs.extend(rendered.segment_proxy_seqs.iter().copied()); + } + protected.segment_proxy_seqs.extend(renderer_candidate_window_proxy_seqs(session)); + + for (proxy_seq, entry) in &session.segments { + if entry.access.active_readers() > 0 + || matches!(entry.status, SegmentCacheStatus::Fetching { .. }) + || (entry.origin_fetch_ref.is_some() + && (matches!(entry.status, SegmentCacheStatus::Queued { .. }) + || session.segment_prefetch_queue.contains(*proxy_seq))) + { + protected.segment_proxy_seqs.insert(*proxy_seq); + } + } + + for proxy_seq in &protected.segment_proxy_seqs { + if let Some(map_ref) = session.segments.get(proxy_seq).and_then(|segment| segment.map_ref) { + protected.map_ids.insert(map_ref); + } + } + for (map_id, map) in &session.maps { + if map.access.active_readers() > 0 + || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) + { + protected.map_ids.insert(*map_id); + } + } + + protected + } +} + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct GarbageCollectionReport { + pub secret_cache_invalidated: bool, + pub secret_cache_invalidation_deferred: bool, + pub temp_files_deleted: usize, + pub orphan_session_dirs_deleted: usize, + pub stale_queue_entries_removed: usize, + pub segments_deleted_duration: usize, + pub segments_deleted_size_session: usize, + pub segments_deleted_size_global: usize, + pub maps_deleted: usize, + pub sessions_deleted: usize, + pub removed_session_ids: Vec, + pub transient_resources_pruned: usize, + pub transient_objects_deleted: usize, + pub transient_object_bytes_deleted: u64, +} + +impl GarbageCollectionReport { + fn segments_deleted(&self) -> usize { + self.segments_deleted_duration + .saturating_add(self.segments_deleted_size_session) + .saturating_add(self.segments_deleted_size_global) + } + + pub fn did_cleanup_or_invalidate(&self) -> bool { + self.secret_cache_invalidated + || self.secret_cache_invalidation_deferred + || self.temp_files_deleted > 0 + || self.orphan_session_dirs_deleted > 0 + || self.stale_queue_entries_removed > 0 + || self.segments_deleted() > 0 + || self.maps_deleted > 0 + || self.sessions_deleted > 0 + || self.transient_resources_pruned > 0 + || self.transient_objects_deleted > 0 + } +} + +pub struct HlsGarbageCollector { + sessions: Arc, + cache: Arc, + policy: ArcSwap, + rewrite_secret_fingerprint: ArcSwap, + metrics: Arc, +} + +impl HlsGarbageCollector { + pub fn new( + sessions: Arc, + cache: Arc, + policy: GarbageCollectionPolicy, + rewrite_secret_fingerprint: String, + ) -> Self { + Self::new_with_metrics( + sessions, + cache, + policy, + rewrite_secret_fingerprint, + Arc::new(HlsCacheMetrics::default()), + ) + } + + pub fn new_with_metrics( + sessions: Arc, + cache: Arc, + policy: GarbageCollectionPolicy, + rewrite_secret_fingerprint: String, + metrics: Arc, + ) -> Self { + Self { + sessions, + cache, + policy: ArcSwap::from_pointee(policy), + rewrite_secret_fingerprint: ArcSwap::from_pointee(rewrite_secret_fingerprint), + metrics, + } + } + + pub fn update_config(&self, policy: GarbageCollectionPolicy, rewrite_secret_fingerprint: String) { + self.policy.store(Arc::new(policy)); + self.rewrite_secret_fingerprint.store(Arc::new(rewrite_secret_fingerprint)); + } + + pub fn policy(&self) -> Arc { self.policy.load_full() } + + pub fn rewrite_secret_fingerprint(&self) -> String { self.rewrite_secret_fingerprint.load().to_string() } + + pub async fn run_once(&self, now_ms: u64) -> io::Result { + let policy = self.policy.load_full(); + let mut report = GarbageCollectionReport::default(); + if self.ensure_cache_marker(&mut report).await? { + self.record_report_metrics(&report); + return Ok(report); + } + + // Captured before the in-memory session snapshot so any directory committed + // after this instant is treated as a potential concurrent create and is + // skipped by the orphan cleanup freshness guard. + let gc_start = SystemTime::now(); + let cutoff = gc_start + .checked_sub(Duration::from_millis(policy.temp_file_retention_ms)) + .unwrap_or(SystemTime::UNIX_EPOCH); + report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?; + + let sessions = self.sessions.list_sessions().await; + let mut active_session_ids = HashSet::with_capacity(sessions.len()); + for session in &sessions { + active_session_ids.insert(session.read().await.proxy_session_id.clone()); + } + report.orphan_session_dirs_deleted = self + .cache + .delete_orphan_session_dirs(&active_session_ids, gc_start) + .await?; + let mut pending_deletions = Vec::new(); + for session in &sessions { + let mut session = session.write().await; + let mut deletions = Self::collect_session_deletions(&mut session, now_ms, &policy, &mut report); + pending_deletions.append(&mut deletions); + } + self.delete_cache_objects(pending_deletions).await?; + + let global_deletions = self.collect_global_size_deletions(&sessions, &policy, &mut report).await; + self.delete_cache_objects(global_deletions).await?; + + for session in &sessions { + self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await?; + } + + self.record_report_metrics(&report); + if report.did_cleanup_or_invalidate() { + info!( + "HLS session garbage collection completed: temp_files_deleted={} orphan_session_dirs_deleted={} stale_queue_entries_removed={} segments_deleted={} maps_deleted={} transient_resources_pruned={} transient_objects_deleted={} transient_object_bytes_deleted={} sessions_deleted={}", + report.temp_files_deleted, + report.orphan_session_dirs_deleted, + report.stale_queue_entries_removed, + report.segments_deleted(), + report.maps_deleted, + report.transient_resources_pruned, + report.transient_objects_deleted, + report.transient_object_bytes_deleted, + report.sessions_deleted, + ); + } + Ok(report) + } + + async fn ensure_cache_marker(&self, report: &mut GarbageCollectionReport) -> io::Result { + let rewrite_secret_fingerprint = self.rewrite_secret_fingerprint.load_full(); + match self.cache.read_rewrite_secret_fingerprint().await? { + Some(current) if current == *rewrite_secret_fingerprint => Ok(false), + Some(_) => { + self.metrics.record_secret_marker_mismatch(); + warn!("HLS rewrite secret changed or cache marker mismatch detected: action=validate-cache-marker"); + match self.cache.invalidate_all_if_no_active_temp_files().await? { + CacheInvalidationOutcome::Invalidated => { + self.sessions.clear().await; + self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; + report.secret_cache_invalidated = true; + info!("HLS rewrite secret changed or cache marker mismatch detected: action=cache-invalidated"); + Ok(true) + } + CacheInvalidationOutcome::DeferredActiveTempFiles => { + report.secret_cache_invalidation_deferred = true; + self.metrics.record_secret_invalidation_deferred(); + warn!( + "HLS rewrite secret changed or cache marker mismatch detected: action=deferred-active-temp-files" + ); + Ok(true) + } + } + } + None => { + self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; + Ok(false) + } + } + } + + fn collect_session_deletions( + session: &mut HlsSession, + now_ms: u64, + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + ) -> Vec { + let mut deletions = Vec::new(); + + report.stale_queue_entries_removed = + report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); + + let transient_before = session.transient.resources.len(); + session.transient.prune_expired(now_ms); + report.transient_resources_pruned = report + .transient_resources_pruned + .saturating_add(transient_before.saturating_sub(session.transient.resources.len())); + + for removal in session.transient.prune_expired_objects(now_ms) { + report.transient_objects_deleted = report.transient_objects_deleted.saturating_add(1); + report.transient_object_bytes_deleted = + report.transient_object_bytes_deleted.saturating_add(removal.content_length); + deletions.push(CacheObjectDeletion::TransientObject(removal.key)); + } + + while let Some(proxy_seq) = + duration_expired_head_segment(session, &ProtectedSet::from_session(session), policy, now_ms) + { + if let Some(deletion) = remove_segment_entry(session, proxy_seq) { + report.segments_deleted_duration = report.segments_deleted_duration.saturating_add(1); + deletions.push(CacheObjectDeletion::Segment(deletion)); + } + } + + let mut session_size = session_cache_size(session); + while session_size > policy.cache_bytes_per_session { + let Some(removal) = session.transient.remove_oldest_ready_object() else { + break; + }; + session_size = session_size.saturating_sub(removal.content_length); + report.transient_objects_deleted = report.transient_objects_deleted.saturating_add(1); + report.transient_object_bytes_deleted = + report.transient_object_bytes_deleted.saturating_add(removal.content_length); + deletions.push(CacheObjectDeletion::TransientObject(removal.key)); + } + while session_size > policy.cache_bytes_per_session { + let Some(candidate) = fifo_head_size_candidate(session, &ProtectedSet::from_session(session)) else { + break; + }; + session_size = session_size.saturating_sub(candidate.content_length); + if let Some(deletion) = remove_segment_entry(session, candidate.proxy_seq) { + report.segments_deleted_size_session = report.segments_deleted_size_session.saturating_add(1); + deletions.push(CacheObjectDeletion::Segment(deletion)); + } + } + + for map_id in unprotected_unreferenced_map_ids(session, &ProtectedSet::from_session(session)) { + if let Some(deletion) = remove_map_entry(session, map_id) { + session_size = session_size.saturating_sub(deletion.content_length); + report.maps_deleted = report.maps_deleted.saturating_add(1); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + } + } + + deletions + } + + async fn collect_global_size_deletions( + &self, + sessions: &[HlsSessionHandle], + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + ) -> Vec { + let mut total_size = total_sessions_cache_size(sessions).await; + let mut deletions = Vec::new(); + + loop { + if total_size <= policy.cache_bytes_global { + break; + } + let Some(candidate) = oldest_global_transient_object_candidate(sessions).await else { + break; + }; + let mut session = candidate.session.write().await; + let Some(removal) = session.transient.remove_oldest_ready_object() else { + continue; + }; + total_size = total_size.saturating_sub(removal.content_length); + report.transient_objects_deleted = report.transient_objects_deleted.saturating_add(1); + report.transient_object_bytes_deleted = + report.transient_object_bytes_deleted.saturating_add(removal.content_length); + deletions.push(CacheObjectDeletion::TransientObject(removal.key)); + } + + loop { + if total_size <= policy.cache_bytes_global { + break; + } + let Some(candidate) = oldest_global_fifo_head_candidate(sessions).await else { + break; + }; + let mut session = candidate.session.write().await; + let Some(current_head) = fifo_head_size_candidate(&session, &ProtectedSet::from_session(&session)) else { + continue; + }; + if current_head.proxy_seq != candidate.proxy_seq { + continue; + } + let Some(deletion) = remove_segment_entry(&mut session, candidate.proxy_seq) else { + continue; + }; + total_size = total_size.saturating_sub(candidate.content_length); + report.segments_deleted_size_global = report.segments_deleted_size_global.saturating_add(1); + deletions.push(CacheObjectDeletion::Segment(deletion)); + + for map_id in unprotected_unreferenced_map_ids(&session, &ProtectedSet::from_session(&session)) { + if let Some(deletion) = remove_map_entry(&mut session, map_id) { + total_size = total_size.saturating_sub(deletion.content_length); + report.maps_deleted = report.maps_deleted.saturating_add(1); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + } + } + } + deletions + } + + async fn remove_idle_session_if_still_idle( + &self, + session: &HlsSessionHandle, + now_ms: u64, + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + ) -> io::Result<()> { + let (key, proxy_session_id) = { + let mut session = session.write().await; + if !Self::should_remove_idle_session(&session, now_ms, policy) { + return Ok(()); + } + session.mark_for_gc_removal(); + (session.key.clone(), session.proxy_session_id.clone()) + }; + + if self.cache.has_active_temp_files_for_session(&proxy_session_id).await { + session.write().await.clear_gc_removal_mark(); + return Ok(()); + } + + if self + .sessions + .remove_session_marking_expired( + &key, + &proxy_session_id, + now_ms, + HlsExpiredSessionReason::SessionIdleTimeout, + None, + ) + .await + .is_some() + { + self.cache.delete_session_dir(&proxy_session_id).await?; + report.sessions_deleted = report.sessions_deleted.saturating_add(1); + report.removed_session_ids.push(proxy_session_id); + } else { + session.write().await.clear_gc_removal_mark(); + } + Ok(()) + } + + async fn delete_cache_objects(&self, deletions: Vec) -> io::Result<()> { + for deletion in deletions { + match deletion { + CacheObjectDeletion::Segment(key) => { + info!( + "Segment '{:06}' removed: session={} source=normal", + key.proxy_seq(), + safe_proxy_session_id(key.proxy_session_id()), + ); + self.cache.delete(&key).await?; + } + CacheObjectDeletion::Map(key) => { + self.cache.delete(&key).await?; + } + CacheObjectDeletion::TransientObject(key) => { + self.cache.delete(&key).await?; + } + } + } + Ok(()) + } + + fn record_report_metrics(&self, report: &GarbageCollectionReport) { + self.metrics.record_gc_run(); + self.metrics.record_segments_removed(report.segments_deleted()); + self.metrics.record_maps_removed(report.maps_deleted); + } + + fn should_remove_idle_session(session: &HlsSession, now_ms: u64, policy: &GarbageCollectionPolicy) -> bool { + session.can_expire_idle_session(now_ms, policy.session_idle_timeout_ms) + } +} + +pub fn build_rewrite_secret_fingerprint(rewrite_secret: &[u8]) -> String { + let digest = Sha256::digest(rewrite_secret); + let value = digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)); + format!("{value:016x}") +} + +pub fn exec_hls_cache_gc(app_state: &Arc, cancel_token: &CancellationToken) { + let hls_proxy = Arc::clone(&app_state.hls_proxy); + let active_users = Arc::clone(&app_state.active_users); + let active_provider = Arc::clone(&app_state.active_provider); + let cancel_token = cancel_token.clone(); + tokio::spawn(async move { + loop { + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + hls_proxy + .sync_all_session_access_leases_and_detach_if_needed(&active_users, &active_provider, now_ms) + .await; + match hls_proxy.run_garbage_collection_once(now_ms).await { + Ok(report) if report.did_cleanup_or_invalidate() => { + debug!( + "HLS cache state snapshot after garbage collection: {}", + hls_proxy.debug_state_summary().await + ); + } + Ok(_) => {} + Err(err) => { + error!("HLS cache garbage collection failed: {err}"); + } + } + tokio::select! { + () = cancel_token.cancelled() => break, + () = tokio::time::sleep(HLS_CACHE_GC_INTERVAL) => {} + } + } + }); +} + +#[derive(Clone)] +enum CacheObjectDeletion { + Segment(SegmentCacheKey), + Map(MapCacheKey), + TransientObject(TransientObjectCacheKey), +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct SegmentDeleteCandidate { + proxy_seq: u64, + content_length: u64, + last_relevant_at_ms: u64, +} + +#[derive(Clone)] +struct GlobalSegmentCandidate { + session: HlsSessionHandle, + proxy_seq: u64, + content_length: u64, + last_relevant_at_ms: u64, +} + +#[derive(Clone)] +struct GlobalTransientObjectCandidate { + session: HlsSessionHandle, + last_accessed_at_ms: u64, +} + +async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 { + let mut total_size = 0_u64; + for session in sessions { + let session = session.read().await; + total_size = total_size.saturating_add(session_cache_size(&session)); + } + total_size +} + +async fn oldest_global_fifo_head_candidate(sessions: &[HlsSessionHandle]) -> Option { + let mut candidates = Vec::new(); + for session in sessions { + let session_guard = session.read().await; + if let Some(candidate) = fifo_head_size_candidate(&session_guard, &ProtectedSet::from_session(&session_guard)) { + candidates.push(GlobalSegmentCandidate { + session: Arc::clone(session), + proxy_seq: candidate.proxy_seq, + content_length: candidate.content_length, + last_relevant_at_ms: candidate.last_relevant_at_ms, + }); + } + } + candidates.into_iter().min_by_key(|candidate| (candidate.last_relevant_at_ms, candidate.proxy_seq)) +} + +async fn oldest_global_transient_object_candidate( + sessions: &[HlsSessionHandle], +) -> Option { + let mut candidates = Vec::new(); + for session in sessions { + let session_guard = session.read().await; + candidates.extend(session_guard.transient.object_cache.values().filter_map(|entry| { + entry.ready_content_length()?; + if entry.access.active_readers() > 0 { + return None; + } + Some(GlobalTransientObjectCandidate { + session: Arc::clone(session), + last_accessed_at_ms: entry.last_accessed_at_ms, + }) + })); + } + candidates.into_iter().min_by_key(|candidate| candidate.last_accessed_at_ms) +} + +fn remove_stale_queue_entries(session: &mut HlsSession) -> usize { + let mut removed = 0_usize; + for proxy_seq in session.segment_prefetch_queue.proxy_seqs() { + let stale = session.segments.get(&proxy_seq).is_none_or(|segment| { + segment.origin_fetch_ref.is_none() || !matches!(segment.status, SegmentCacheStatus::Queued { .. }) + }); + if stale && session.segment_prefetch_queue.remove(proxy_seq).is_some() { + removed = removed.saturating_add(1); + } + } + removed +} + +fn duration_expired_head_segment( + session: &HlsSession, + protected: &ProtectedSet, + policy: &GarbageCollectionPolicy, + now_ms: u64, +) -> Option { + let (proxy_seq, segment) = session.segments.iter().next()?; + if protected.segment_proxy_seqs.contains(proxy_seq) { + return None; + } + let last_relevant_at_ms = segment_last_relevant_at_ms(segment)?; + let retention_ms = match segment.status { + SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::FailedPermanent { .. } + | SegmentCacheStatus::Expired => policy.failed_segment_retention_ms, + SegmentCacheStatus::Ready { .. } => policy + .cache_duration_ms + .max(segment.duration_ms.saturating_add(session.longest_rendered_playlist_duration_ms)), + SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. } => { + return None + } + }; + (now_ms.saturating_sub(last_relevant_at_ms) >= retention_ms).then_some(*proxy_seq) +} + +fn fifo_head_size_candidate(session: &HlsSession, protected: &ProtectedSet) -> Option { + let (proxy_seq, segment) = session.segments.iter().next()?; + if protected.segment_proxy_seqs.contains(proxy_seq) { + return None; + } + let SegmentCacheStatus::Ready { content_length, .. } = segment.status else { + return None; + }; + Some(SegmentDeleteCandidate { + proxy_seq: *proxy_seq, + content_length, + last_relevant_at_ms: segment_last_relevant_at_ms(segment).unwrap_or_default(), + }) +} + +fn segment_last_relevant_at_ms(segment: &super::SegmentEntry) -> Option { + let status_at = match segment.status { + SegmentCacheStatus::Ready { ready_at_ms, .. } => Some(ready_at_ms), + SegmentCacheStatus::FailedRetryable { failed_at_ms, .. } + | SegmentCacheStatus::FailedPermanent { failed_at_ms, .. } => Some(failed_at_ms), + SegmentCacheStatus::Expired => segment.last_rendered_at_ms, + SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. } => { + None + } + }; + [status_at, segment.last_rendered_at_ms, Some(segment.access.last_accessed_at_ms()).filter(|value| *value > 0)] + .into_iter() + .flatten() + .max() +} + +fn session_cache_size(session: &HlsSession) -> u64 { + let segment_bytes = session + .segments + .values() + .map(|segment| match segment.status { + SegmentCacheStatus::Ready { content_length, .. } => content_length, + _ => 0, + }) + .sum::(); + let map_bytes = session + .maps + .values() + .map(|map| match map.status { + MapCacheStatus::Ready { content_length, .. } => content_length, + _ => 0, + }) + .sum::(); + segment_bytes.saturating_add(map_bytes).saturating_add(session.transient.ready_object_cache_size()) +} + +fn remove_segment_entry(session: &mut HlsSession, proxy_seq: u64) -> Option { + let segment = session.segments.remove(&proxy_seq)?; + session.segment_prefetch_queue.remove(proxy_seq); + session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq); + if segment.discontinuity_before { + session.discontinuity_sequence = session.discontinuity_sequence.saturating_add(1); + } + Some(segment.cache_key) +} + +fn unprotected_unreferenced_map_ids(session: &HlsSession, protected: &ProtectedSet) -> Vec { + let referenced = session.segments.values().filter_map(|segment| segment.map_ref).collect::>(); + session + .maps + .iter() + .filter_map(|(map_id, map)| { + if referenced.contains(map_id) + || protected.map_ids.contains(map_id) + || map.access.active_readers() > 0 + || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) + { + return None; + } + Some(*map_id) + }) + .collect() +} + +struct MapEntryDeletion { + key: MapCacheKey, + content_length: u64, +} + +fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) -> Option { + let map = session.maps.remove(&map_id)?; + let content_length = match map.status { + MapCacheStatus::Ready { content_length, .. } => content_length, + _ => 0, + }; + session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id); + Some(MapEntryDeletion { key: map.cache_key, content_length }) +} + +#[cfg(test)] +mod tests { + use super::{ + build_rewrite_secret_fingerprint, GarbageCollectionPolicy, GarbageCollectionReport, HlsGarbageCollector, + }; + use crate::{ + api::model::{ + HlsSegmentCache, HlsSessionKey, HlsSessionStore, MapCacheStatus, OriginMapKey, ProxyMapId, + SegmentCacheStatus, SegmentFetchPriority, TransientPassthroughState, TransientResourceId, + }, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + }; + use std::sync::Arc; + use tokio::io::AsyncWriteExt; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + fn test_policy() -> GarbageCollectionPolicy { + GarbageCollectionPolicy { + cache_duration_ms: 300, + cache_bytes_global: 10_000, + cache_bytes_per_session: 10_000, + session_idle_timeout_ms: 1_000, + temp_file_retention_ms: 30_000, + failed_segment_retention_ms: 10, + } + } + + async fn gc_with_session( + temp_dir: &tempfile::TempDir, + ) -> (HlsGarbageCollector, crate::api::model::HlsSessionHandle) { + let sessions = Arc::new(HlsSessionStore::new()); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + let gc = HlsGarbageCollector::new(sessions, cache, test_policy(), build_rewrite_secret_fingerprint(b"secret")); + (gc, session) + } + + fn update_gc_policy(gc: &HlsGarbageCollector, update: impl FnOnce(&mut GarbageCollectionPolicy)) { + let mut policy = gc.policy().as_ref().clone(); + update(&mut policy); + gc.update_config(policy, gc.rewrite_secret_fingerprint()); + } + + fn six_segment_manifest() -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n", + ) + } + + fn apply_six_segment_manifest_for_gc(session: &mut super::HlsSession) { + session.proxy_next_seq = Some(1); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + } + + #[test] + fn gc_report_is_quiet_when_nothing_changed() { + let report = GarbageCollectionReport::default(); + + assert!(!report.did_cleanup_or_invalidate()); + } + + #[test] + fn gc_report_logs_when_cleanup_or_invalidation_happened() { + let mut report = GarbageCollectionReport { temp_files_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { stale_queue_entries_removed: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { segments_deleted_duration: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { maps_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { sessions_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { transient_resources_pruned: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { secret_cache_invalidated: true, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = + GarbageCollectionReport { secret_cache_invalidation_deferred: true, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + } + + async fn populate_ready_segments( + gc: &HlsGarbageCollector, + session: &crate::api::model::HlsSessionHandle, + ready_at_ms: u64, + ) { + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for segment in session.segments.values_mut() { + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + session.render_and_store_manifest(ready_at_ms).expect("ready manifest should render"); + } + } + + #[tokio::test] + async fn gc_keeps_last_rendered_manifest_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + populate_ready_segments(&gc, &session, 0).await; + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert_eq!(session.read().await.segments.len(), 6); + } + + #[tokio::test] + async fn gc_keeps_active_readers() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + segment.access.reader_started(1); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); + } + + #[tokio::test] + async fn gc_keeps_fetching_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::Prefetch, started_at_ms: 1 }; + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); + } + + #[tokio::test] + async fn duration_gc_deletes_old_unprotected_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 1); + assert!(!session.read().await.segments.contains_key(&1)); + } + + #[tokio::test] + async fn duration_gc_stops_at_protected_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + session.segments.get_mut(&1).expect("head segment").access.reader_started(1); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + } + + #[tokio::test] + async fn duration_gc_stops_at_not_expired_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for (proxy_seq, ready_at_ms) in [(1, 9_950), (2, 0)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + } + + #[tokio::test] + async fn session_size_gc_deletes_oldest_unprotected_segment() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_session, 2); + let session = session.read().await; + assert!(!session.segments.contains_key(&1)); + assert!(!session.segments.contains_key(&2)); + assert!(session.segments.contains_key(&3)); + } + + #[tokio::test] + async fn session_size_gc_stops_at_protected_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + session.segments.get_mut(&1).expect("head segment").access.reader_started(1); + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_session, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + assert!(session.segments.contains_key(&3)); + } + + #[tokio::test] + async fn protected_map_remains_and_unreferenced_map_is_deleted() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n", + ); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + let protected_map_id = ProxyMapId(0); + session.maps.get_mut(&protected_map_id).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + let unreferenced_key = OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "http://origin.example.com/live/unused.mp4".to_string(), + byte_range: None, + }; + let unreferenced_map = crate::api::model::MapEntry::new( + &session.proxy_session_id, + ProxyMapId(1), + unreferenced_key.clone(), + "mp4".to_string(), + ); + session.maps.insert(ProxyMapId(1), unreferenced_map); + session.origin_map_to_proxy.insert(unreferenced_key, ProxyMapId(1)); + session.maps.get_mut(&ProxyMapId(1)).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + session.render_and_store_manifest(1).expect("manifest should render"); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.maps_deleted, 1); + let session = session.read().await; + assert!(session.maps.contains_key(&ProxyMapId(0))); + assert!(!session.maps.contains_key(&ProxyMapId(1))); + } + + #[tokio::test] + async fn map_referenced_by_remaining_segment_survives_fifo_gc() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let manifest = + normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXTINF:4.0,\n1.m4s\n"); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.maps.get_mut(&ProxyMapId(0)).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + for (proxy_seq, ready_at_ms) in [(0, 0), (1, 9_950)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 1); + assert_eq!(report.maps_deleted, 0); + let session = session.read().await; + assert!(!session.segments.contains_key(&0)); + assert!(session.segments.contains_key(&1)); + assert!(session.maps.contains_key(&ProxyMapId(0))); + } + + #[tokio::test] + async fn global_size_gc_subtracts_map_bytes_after_segment_removal() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 25); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init0.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4.0,\n1.m4s\n", + ); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + session.maps.get_mut(&ProxyMapId(0)).expect("first map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + session.maps.get_mut(&ProxyMapId(1)).expect("second map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 1 }; + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 1); + assert_eq!(report.maps_deleted, 1); + let session = session.read().await; + assert!(!session.segments.contains_key(&0)); + assert!(session.segments.contains_key(&1)); + assert!(!session.maps.contains_key(&ProxyMapId(0))); + assert!(session.maps.contains_key(&ProxyMapId(1))); + } + + #[tokio::test] + async fn transient_resource_mappings_expire_unless_active() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let resource_id = { + let mut session = session.write().await; + let resource = crate::api::model::TransientResourceRef::new( + crate::api::model::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + resource_id + }; + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_resources_pruned, 1); + assert!(!session.read().await.transient.resources.contains_key(&resource_id)); + } + + #[tokio::test] + async fn transient_resource_mappings_in_last_manifest_are_protected() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let resource_id = { + let mut session = session.write().await; + let resource = crate::api::model::TransientResourceRef::new( + crate::api::model::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + session.transient.replace_manifest( + format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n", resource_id.0), + 0, + ); + resource_id + }; + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_resources_pruned, 0); + assert!(session.read().await.transient.resources.contains_key(&resource_id)); + } + + #[tokio::test] + async fn transient_object_cache_expires_by_cache_policy() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let cache_key = { + let mut session = session.write().await; + let key = TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &TransientResourceId("object1".to_string()), + "ts", + ); + gc.cache.write_bytes_and_commit(&key, b"transient-body").await.expect("object writes"); + session.transient.mark_object_ready(&key, "video/mp2t".to_string(), 14, 0, 10); + key + }; + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_objects_deleted, 1); + assert_eq!(report.transient_object_bytes_deleted, 14); + assert!(gc.cache.metadata(&cache_key).await.expect("metadata read").is_none()); + } + + #[tokio::test] + async fn session_size_gc_deletes_transient_objects_before_timeline_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("segment writes"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 100 }; + let key = TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &TransientResourceId("object1".to_string()), + "ts", + ); + gc.cache.write_bytes_and_commit(&key, b"transient-body").await.expect("object writes"); + session.transient.mark_object_ready(&key, "video/mp2t".to_string(), 14, 100, 10_000); + key + }; + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.transient_objects_deleted, 1); + assert_eq!(report.segments_deleted_size_session, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(!session.transient.object_cache.contains_key(&cache_key)); + } + + #[tokio::test] + async fn secret_fingerprint_mismatch_invalidates_cache_and_sessions() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + } + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert!(report.secret_cache_invalidated); + assert!(!report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 0); + let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); + assert_eq!( + gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), + Some(rewrite_secret_fingerprint.as_str()) + ); + } + + #[tokio::test] + async fn secret_fingerprint_mismatch_with_active_temp_defers_invalidation() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get(&1).expect("segment").cache_key.clone() + }; + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + let (mut writer, reader) = tokio::io::duplex(64); + let cache = Arc::clone(&gc.cache); + let task_key = cache_key.clone(); + let write_task = tokio::spawn(async move { cache.write_temp_and_commit(&task_key, reader).await }); + for _ in 0..50 { + if gc.cache.has_active_temp_files().await { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(gc.cache.has_active_temp_files().await); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert!(!report.secret_cache_invalidated); + assert!(report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 1); + assert_eq!(gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), Some("mismatch")); + assert!(gc.cache.has_active_temp_files().await); + writer.write_all(b"done").await.expect("write temp body"); + drop(writer); + write_task.await.expect("temp write task joins").expect("temp write commits"); + assert!(gc.cache.metadata(&cache_key).await.expect("metadata should read").is_some()); + } + + #[tokio::test] + async fn deferred_secret_fingerprint_invalidation_runs_after_temp_commit() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get(&1).expect("segment").cache_key.clone() + }; + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + let (mut writer, reader) = tokio::io::duplex(64); + let cache = Arc::clone(&gc.cache); + let task_key = cache_key.clone(); + let write_task = tokio::spawn(async move { cache.write_temp_and_commit(&task_key, reader).await }); + for _ in 0..50 { + if gc.cache.has_active_temp_files().await { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(gc.run_once(1).await.expect("first gc should run").secret_cache_invalidation_deferred); + writer.write_all(b"done").await.expect("write temp body"); + drop(writer); + write_task.await.expect("temp write task joins").expect("temp write commits"); + + let report = gc.run_once(2).await.expect("second gc should run"); + + assert!(report.secret_cache_invalidated); + assert!(!report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 0); + assert_eq!(gc.cache.metadata(&cache_key).await.expect("metadata should read"), None); + let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); + assert_eq!( + gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), + Some(rewrite_secret_fingerprint.as_str()) + ); + } + + #[tokio::test] + async fn global_size_gc_deletes_oldest_unprotected_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; + + for session in [&first_session, &second_session] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 4); + let remaining_size = { + let first = first_session.read().await; + let second = second_session.read().await; + super::session_cache_size(&first) + super::session_cache_size(&second) + }; + assert!(remaining_size <= 24); + } + + #[tokio::test] + async fn global_size_gc_uses_only_current_fifo_heads() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; + + for session in [&first_session, &second_session] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + first_session.write().await.segments.get_mut(&1).expect("head segment").access.reader_started(1); + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 3); + let first = first_session.read().await; + assert!(first.segments.contains_key(&1)); + assert!(first.segments.contains_key(&2)); + assert!(first.segments.contains_key(&3)); + let second = second_session.read().await; + assert!(!second.segments.contains_key(&1)); + assert!(!second.segments.contains_key(&2)); + assert!(!second.segments.contains_key(&3)); + } + + #[tokio::test] + async fn temp_file_gc_deletes_old_tmp_files() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let temp_path = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let path = gc.cache.object_path(&session.segments.get(&1).expect("segment").cache_key); + let parent = path.parent().expect("cache object has parent"); + tokio::fs::create_dir_all(parent).await.expect("parent dir"); + parent.join("000001.ts.tmp.old") + }; + tokio::fs::write(&temp_path, b"partial").await.expect("temp write"); + let old_time = filetime::FileTime::from_unix_time(1, 0); + filetime::set_file_mtime(&temp_path, old_time).expect("set mtime"); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert_eq!(report.temp_files_deleted, 1); + assert!(!temp_path.exists()); + } + + #[tokio::test] + async fn session_gc_keeps_idle_session_with_active_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (cache_key, proxy_session_id) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + (session.segments.get(&1).expect("segment").cache_key.clone(), session.proxy_session_id.clone()) + }; + let (mut writer, reader) = tokio::io::duplex(64); + let cache = Arc::clone(&gc.cache); + let write_task = tokio::spawn(async move { cache.write_temp_and_commit(&cache_key, reader).await }); + for _ in 0..50 { + if gc.cache.has_active_temp_files_for_session(&proxy_session_id).await { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!(gc.cache.has_active_temp_files_for_session(&proxy_session_id).await); + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 0); + assert!(!session.read().await.is_gc_marked_for_removal()); + assert_eq!(gc.sessions.len().await, 1); + writer.write_all(b"done").await.expect("write temp body"); + drop(writer); + write_task.await.expect("temp write task joins").expect("temp write commits"); + } + + #[tokio::test] + async fn session_gc_removes_idle_session_without_activity() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (key, proxy_session_id) = { + let session = session.read().await; + (session.key.clone(), session.proxy_session_id.clone()) + }; + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 1); + assert!(gc.sessions.get_by_key(&key).await.is_none()); + assert!(gc.sessions.get_by_proxy_session_id(&proxy_session_id).await.is_none()); + } + + #[tokio::test] + async fn session_gc_final_recheck_keeps_new_activity() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + session.write().await.last_client_access_at_ms = 1_500; + let mut report = super::GarbageCollectionReport::default(); + let policy = gc.policy(); + + gc.remove_idle_session_if_still_idle(&session, 2_000, &policy, &mut report) + .await + .expect("session removal check should run"); + + assert_eq!(report.sessions_deleted, 0); + assert_eq!(gc.sessions.len().await, 1); + assert!(!session.read().await.is_gc_marked_for_removal()); + } + + #[tokio::test] + async fn active_transient_resource_reader_protects_idle_session() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + let resource = crate::api::model::TransientResourceRef::new( + crate::api::model::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + resource.access.reader_started(1); + session.transient.upsert_resources([resource]); + } + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 0); + assert_eq!(gc.sessions.len().await, 1); + } +} diff --git a/backend/src/api/model/hls_cache/headers.rs b/backend/src/api/model/hls_cache/headers.rs new file mode 100644 index 000000000..1aa7c4c0a --- /dev/null +++ b/backend/src/api/model/hls_cache/headers.rs @@ -0,0 +1,212 @@ +use crate::api::model::proxy::header_policy::{HeaderProtocol, HopByHopHeader}; +use crate::model::ReverseProxyDisabledHeaderConfig; +use axum::http::{header, HeaderMap, HeaderName, HeaderValue}; +use std::collections::HashMap; + +/// Returns true when a header must never be forwarded by the live HLS cache proxy. +/// +/// Thin wrapper around `HopByHopHeader::is_sensitive(HeaderProtocol::Hls, …)` so the +/// hard-coded hop-by-hop list and Tuliprox-internal prefix live in one place shared +/// with the MPEG-TS reverse-proxy path. Adding a new "always strip" header now +/// requires editing only `proxy/header_policy.rs`. +pub fn should_remove_hls_origin_header( + header_name: &str, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, +) -> bool { + HopByHopHeader::is_sensitive(header_name, HeaderProtocol::Hls, disabled_headers) +} + +/// Removes disabled and sensitive headers before an origin request leaves Tuliprox. +pub fn scrub_hls_origin_headers(headers: &mut HeaderMap, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>) { + let mut names = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .collect::>(); + names.extend(headers + .keys() + .filter(|name| should_remove_hls_origin_header(name.as_str(), disabled_headers)) + .cloned() + ); + for name in names { + headers.remove(name); + } +} + +pub fn sanitized_hls_origin_headers( + source_headers: &HeaderMap, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, +) -> HeaderMap { + let mut headers = source_headers.clone(); + scrub_hls_origin_headers(&mut headers, disabled_headers); + headers +} + +/// Extracts trusted provider session cookies from origin response headers. +pub fn extract_hls_provider_session_header_map(headers: &HeaderMap) -> HeaderMap { + let cookies = headers + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .filter_map(|value| value.split(';').next().map(str::trim)) + .filter(|value| !value.is_empty()) + .collect::>(); + + let mut session_headers = HeaderMap::new(); + if let Some(cookie_header) = (!cookies.is_empty()) + .then(|| cookies.join("; ")) + .and_then(|value| HeaderValue::from_str(&value).ok()) + { + session_headers.insert(header::COOKIE, cookie_header); + } + session_headers +} + +/// Extracts provider session cookies for the legacy `ActiveUserManager` session store. +pub fn extract_hls_provider_session_headers(headers: &HeaderMap) -> HashMap { + extract_hls_provider_session_header_map(headers) + .iter() + .filter_map(|(name, value)| value.to_str().ok().map(|value| (name.as_str().to_string(), value.to_string()))) + .collect() +} + +/// Appends trusted provider session headers after client-header scrubbing. +pub fn append_hls_provider_session_headers(headers: &mut HeaderMap, provider_session_headers: &HeaderMap) { + if let Some(cookie) = provider_session_headers.get(header::COOKIE).cloned() { + headers.insert(header::COOKIE, cookie); + } +} + +pub fn hls_origin_headers_with_provider_session( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, +) -> HeaderMap { + let mut headers = source_headers.clone(); + scrub_hls_origin_headers(&mut headers, None); + append_hls_provider_session_headers(&mut headers, provider_session_headers); + headers +} + +pub fn force_identity_without_range(headers: &mut HeaderMap) { + headers.remove(header::RANGE); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("identity")); +} + +#[cfg(test)] +mod tests { + use super::{ + append_hls_provider_session_headers, extract_hls_provider_session_header_map, + extract_hls_provider_session_headers, force_identity_without_range, scrub_hls_origin_headers, + should_remove_hls_origin_header, + }; + use crate::model::ReverseProxyDisabledHeaderConfig; + use axum::http::{header, HeaderMap, HeaderName, HeaderValue}; + + #[test] + fn should_remove_sensitive_and_disabled_headers() { + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: true, + x_header: true, + cloudflare_header: true, + custom_header: vec!["X-Origin-Secret".to_string()], + }; + + assert!(should_remove_hls_origin_header("Authorization", Some(&disabled))); + assert!(should_remove_hls_origin_header("Cookie", Some(&disabled))); + assert!(should_remove_hls_origin_header("Connection", Some(&disabled))); + assert!(should_remove_hls_origin_header("TE", Some(&disabled))); + assert!(should_remove_hls_origin_header("Trailer", Some(&disabled))); + assert!(should_remove_hls_origin_header("Transfer-Encoding", Some(&disabled))); + assert!(should_remove_hls_origin_header("Upgrade", Some(&disabled))); + assert!(should_remove_hls_origin_header("Proxy-Authorization", Some(&disabled))); + assert!(should_remove_hls_origin_header("Host", Some(&disabled))); + assert!(should_remove_hls_origin_header("X-Tuliprox-Main-Revision", Some(&disabled))); + assert!(should_remove_hls_origin_header("Referer", Some(&disabled))); + assert!(should_remove_hls_origin_header("X-Blocked", Some(&disabled))); + assert!(should_remove_hls_origin_header("CF-Ray", Some(&disabled))); + assert!(should_remove_hls_origin_header("x-origin-secret", Some(&disabled))); + assert!(!should_remove_hls_origin_header("Accept-Language", Some(&disabled))); + } + + #[test] + fn scrub_removes_sensitive_and_disabled_headers_from_header_map() { + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: true, + cloudflare_header: false, + custom_header: vec!["X-Origin-Secret".to_string()], + }; + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("blocked")); + headers.insert(HeaderName::from_static("x-origin-secret"), HeaderValue::from_static("secret")); + headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + + scrub_hls_origin_headers(&mut headers, Some(&disabled)); + + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key("proxy-authorization")); + assert!(!headers.contains_key(header::HOST)); + assert!(!headers.contains_key("x-blocked")); + assert!(!headers.contains_key("x-origin-secret")); + assert_eq!(headers.get(header::ACCEPT_LANGUAGE).expect("language"), "de"); + } + + #[test] + fn scrub_removes_headers_named_by_connection() { + let mut headers = HeaderMap::new(); + headers.insert(header::CONNECTION, HeaderValue::from_static("keep-alive, x-origin-hop")); + headers.insert(HeaderName::from_static("x-origin-hop"), HeaderValue::from_static("secret")); + headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + + scrub_hls_origin_headers(&mut headers, None); + + assert!(!headers.contains_key(header::CONNECTION)); + assert!(!headers.contains_key("x-origin-hop")); + assert!(headers.contains_key(header::ACCEPT_LANGUAGE)); + } + + #[test] + fn identity_helper_removes_existing_range() { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + + force_identity_without_range(&mut headers); + + assert!(!headers.contains_key(header::RANGE)); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); + } + + #[test] + fn extract_provider_session_headers_converts_set_cookie_to_cookie_header() { + let mut headers = HeaderMap::new(); + headers.append(header::SET_COOKIE, HeaderValue::from_static("sid=abc; Path=/; HttpOnly")); + headers.append(header::SET_COOKIE, HeaderValue::from_static("pref=1; SameSite=Lax")); + + let header_map = extract_hls_provider_session_header_map(&headers); + assert_eq!(header_map.get(header::COOKIE).expect("cookie"), "sid=abc; pref=1"); + + let legacy_headers = extract_hls_provider_session_headers(&headers); + assert_eq!(legacy_headers.get("cookie").map(String::as_str), Some("sid=abc; pref=1")); + } + + #[test] + fn append_provider_session_headers_restores_trusted_cookie_after_scrub() { + let mut headers = HeaderMap::new(); + headers.insert(header::COOKIE, HeaderValue::from_static("client=secret")); + scrub_hls_origin_headers(&mut headers, None); + + let mut provider_headers = HeaderMap::new(); + provider_headers.insert(header::COOKIE, HeaderValue::from_static("sid=abc")); + append_hls_provider_session_headers(&mut headers, &provider_headers); + + assert_eq!(headers.get(header::COOKIE).expect("cookie"), "sid=abc"); + } +} diff --git a/backend/src/api/model/hls_cache/ids.rs b/backend/src/api/model/hls_cache/ids.rs new file mode 100644 index 000000000..38af01306 --- /dev/null +++ b/backend/src/api/model/hls_cache/ids.rs @@ -0,0 +1,95 @@ +use base64::{engine::general_purpose, Engine as _}; +use serde::{Deserialize, Serialize}; + +const PROXY_SESSION_ID_LEN: usize = 22; +const PROXY_SESSION_ID_KEY_CONTEXT: &str = "tuliprox:hls-cache:proxy-session-id-key:v1"; + +/// Stable Tuliprox content identity for a live HLS source. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct HlsSessionKey { + pub input_id: u16, + pub stream_ref: String, +} + +impl HlsSessionKey { + pub fn new(input_id: u16, stream_ref: impl Into) -> Self { + Self { input_id, stream_ref: stream_ref.into() } + } + + pub fn canonical(&self) -> String { format!("input:{}|hls|{}", self.input_id, self.stream_ref) } + + pub fn stable_value(&self) -> String { self.canonical() } +} + +/// Public opaque lookup token for HLS proxy URLs. +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct ProxySessionId(pub String); + +/// Builds the public opaque proxy session token from the stable session key. +pub fn build_proxy_session_id(key: &HlsSessionKey, reverse_proxy_rewrite_secret: &[u8]) -> ProxySessionId { + let hls_session_key = blake3::derive_key(PROXY_SESSION_ID_KEY_CONTEXT, reverse_proxy_rewrite_secret); + let digest = blake3::keyed_hash(&hls_session_key, key.stable_value().as_bytes()); + let token = general_purpose::URL_SAFE_NO_PAD.encode(digest.as_bytes()); + ProxySessionId(token.chars().take(PROXY_SESSION_ID_LEN).collect()) +} + +#[cfg(test)] +mod tests { + use super::{build_proxy_session_id, HlsSessionKey}; + + #[test] + fn live_hls_session_key_uses_tuliprox_input_and_stream_ref() { + let first = HlsSessionKey::new(7, "80510"); + let second = HlsSessionKey::new(7, "80510"); + + assert_eq!(first, second); + assert_eq!(first.stable_value(), "input:7|hls|80510"); + } + + #[test] + fn live_hls_session_key_changes_for_different_input_or_stream_ref() { + let first = HlsSessionKey::new(7, "80510"); + let different_input = HlsSessionKey::new(8, "80510"); + let different_stream = HlsSessionKey::new(7, "80511"); + + assert_ne!(first, different_input); + assert_ne!(first, different_stream); + } + + #[test] + fn live_hls_session_key_does_not_contain_origin_or_provider_url_parts() { + let key = HlsSessionKey::new(7, "80510"); + let stable = key.stable_value(); + + assert!(!stable.contains("provider://")); + assert!(!stable.contains("origin.example.com")); + assert!(!stable.contains("user")); + assert!(!stable.contains("pass")); + assert!(!stable.contains(".m3u8")); + } + + #[test] + fn proxy_session_id_is_stable_for_same_key_and_secret() { + let key = HlsSessionKey::new(7, "80510"); + let secret = b"0011223344556677"; + + assert_eq!(build_proxy_session_id(&key, secret), build_proxy_session_id(&key, secret)); + } + + #[test] + fn proxy_session_id_changes_for_different_secret() { + let key = HlsSessionKey::new(7, "80510"); + + assert_ne!( + build_proxy_session_id(&key, b"0011223344556677"), + build_proxy_session_id(&key, b"8899aabbccddeeff") + ); + } + + #[test] + fn proxy_session_id_is_truncated_to_opaque_token_length() { + let key = HlsSessionKey::new(7, "80510"); + + assert_eq!(build_proxy_session_id(&key, b"0011223344556677").0.len(), 22); + } +} diff --git a/backend/src/api/model/hls_cache/lease.rs b/backend/src/api/model/hls_cache/lease.rs new file mode 100644 index 000000000..aaede6cab --- /dev/null +++ b/backend/src/api/model/hls_cache/lease.rs @@ -0,0 +1,1270 @@ +use super::{HlsEffectiveOriginAcquirePolicy, ProxySessionId}; +use crate::api::model::ConnectionKind; +use axum::http::StatusCode; +use base64::{engine::general_purpose, Engine as _}; +use rand::{rngs::OsRng, RngCore, TryRngCore}; +use std::{collections::HashMap, fmt}; + +const HLS_ACCESS_LEASE_ID_BYTES: usize = 16; + +/// Short opaque lookup key for a server-side HLS access lease. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct HlsAccessLeaseId(pub String); + +impl fmt::Debug for HlsAccessLeaseId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("HlsAccessLeaseId").field(&"").finish() + } +} + +/// Stable user/player family used only for diagnostics or future UX grouping. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct HlsPlaybackFamilyKey { + pub username: String, + pub client_fingerprint: String, +} + +impl HlsPlaybackFamilyKey { + pub fn new(username: impl Into, client_fingerprint: impl Into) -> Self { + Self { username: username.into(), client_fingerprint: client_fingerprint.into() } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessLeaseState { + Pending, + Activated, + Idle, + Expired, + Denied, +} + +impl HlsAccessLeaseState { + pub const fn as_log_value(self) -> &'static str { + match self { + Self::Pending => "Pending", + Self::Activated => "Activated", + Self::Idle => "Idle", + Self::Expired => "Expired", + Self::Denied => "Denied", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessLeaseResponseFlag { + ChannelUnavailable { reason: HlsAccessLeaseChannelUnavailableReason, set_at_ms: u64 }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessLeaseChannelUnavailableReason { + OriginAccountUnavailable, + ManifestCommitFailed { reason: HlsFreshManifestRequiredReason }, + ManifestTemporaryFailureThreshold { failures: u32, threshold: u32 }, + SegmentPermanentFailure { status: Option }, + SegmentTemporaryFailureThreshold { failures: u32, threshold: u32 }, + MapPermanentFailure { status: Option }, + TransientObjectPermanentFailure { status: Option }, + TransientObjectTemporaryFailureThreshold { failures: u32, threshold: u32 }, + ResourceWaitThresholdExceeded, +} + +/// Explains why a canonical HLS request required a newly committed manifest. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsFreshManifestRequiredReason { + ColdStart, + ExpiredRevalidation, + PreviousHardManifestFailure, + ProvisioningHandoff, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct HlsAccessLeaseTiming { + pub active_window_ms: u64, + pub valid_window_ms: u64, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessLeasePendingDeadline { + Bootstrap { deadline_ms: u64 }, + FollowUp { deadline_ms: u64 }, +} + +impl HlsAccessLeasePendingDeadline { + pub const fn deadline_ms(self) -> u64 { + match self { + Self::Bootstrap { deadline_ms } | Self::FollowUp { deadline_ms } => deadline_ms, + } + } + + const fn tightened_with(self, candidate: Self) -> Self { + let deadline_ms = if self.deadline_ms() <= candidate.deadline_ms() { + self.deadline_ms() + } else { + candidate.deadline_ms() + }; + match (self, candidate) { + (Self::FollowUp { .. }, _) | (_, Self::FollowUp { .. }) => Self::FollowUp { deadline_ms }, + (Self::Bootstrap { .. }, Self::Bootstrap { .. }) => Self::Bootstrap { deadline_ms }, + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsAccessLease { + pub lease_id: HlsAccessLeaseId, + pub family_key: HlsPlaybackFamilyKey, + pub proxy_session_id: ProxySessionId, + pub username: String, + pub user_session_token: String, + pub input_id: u16, + pub stream_ref: String, + pub virtual_id: u32, + pub origin_connection_kind: ConnectionKind, + pub origin_priority: i8, + pub state: HlsAccessLeaseState, + pub issued_at_ms: u64, + pub last_seen_at_ms: u64, + pub active_until_ms: Option, + pub pending_deadline: Option, + pub valid_until_ms: u64, + pub response_flag: Option, +} + +impl HlsAccessLease { + #[allow(clippy::too_many_arguments)] + pub fn pending( + lease_id: HlsAccessLeaseId, + family_key: HlsPlaybackFamilyKey, + proxy_session_id: ProxySessionId, + username: String, + user_session_token: String, + input_id: u16, + stream_ref: String, + virtual_id: u32, + now_ms: u64, + valid_window_ms: u64, + ) -> Self { + Self { + lease_id, + family_key, + proxy_session_id, + username, + user_session_token, + input_id, + stream_ref, + virtual_id, + origin_connection_kind: ConnectionKind::Normal, + origin_priority: 0, + state: HlsAccessLeaseState::Pending, + issued_at_ms: now_ms, + last_seen_at_ms: now_ms, + active_until_ms: None, + pending_deadline: Some(HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(valid_window_ms), + }), + valid_until_ms: now_ms.saturating_add(valid_window_ms), + response_flag: None, + } + } + + pub const fn with_origin_acquire_policy(mut self, connection_kind: ConnectionKind, priority: i8) -> Self { + self.origin_connection_kind = connection_kind; + self.origin_priority = priority; + self + } + + pub fn update_origin_acquire_policy(&mut self, connection_kind: ConnectionKind, priority: i8) { + self.origin_connection_kind = connection_kind; + self.origin_priority = priority; + } + + pub fn age_ms(&self, now_ms: u64) -> u64 { now_ms.saturating_sub(self.issued_at_ms) } + + pub fn pending_deadline_ms(&self) -> Option { + self.pending_deadline.map(HlsAccessLeasePendingDeadline::deadline_ms) + } + + fn validity_due_at_ms(&self) -> u64 { + if self.state == HlsAccessLeaseState::Pending { + self.pending_deadline_ms().unwrap_or(self.valid_until_ms) + } else { + self.valid_until_ms + } + } + + fn apply_pending_deadline(&mut self, deadline: HlsAccessLeasePendingDeadline) -> bool { + let previous = self.pending_deadline; + let deadline = self.pending_deadline.map_or(deadline, |current| current.tightened_with(deadline)); + self.pending_deadline = Some(deadline); + self.valid_until_ms = deadline.deadline_ms(); + previous != self.pending_deadline + } + + fn refresh_validity(&mut self, now_ms: u64) { + if self.validity_due_at_ms() <= now_ms { + self.state = HlsAccessLeaseState::Expired; + } + } + + fn refresh_activity(&mut self, now_ms: u64) -> Option { + let previous_state = self.state; + self.refresh_validity(now_ms); + if self.state == HlsAccessLeaseState::Expired + && matches!(previous_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) + { + return Some(HlsAccessLeaseIdleRelease { + lease_id: self.lease_id.clone(), + username: self.username.clone(), + user_session_token: self.user_session_token.clone(), + }); + } + if self.state == HlsAccessLeaseState::Activated + && self.active_until_ms.is_some_and(|active_until| active_until <= now_ms) + { + self.state = HlsAccessLeaseState::Idle; + return Some(HlsAccessLeaseIdleRelease { + lease_id: self.lease_id.clone(), + username: self.username.clone(), + user_session_token: self.user_session_token.clone(), + }); + } + None + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum HlsAccessLeaseActivation { + Activated { lease: Box, previous_state: HlsAccessLeaseState }, + Expired, + Denied, + UnknownLease, + SessionMismatch, +} + +impl HlsAccessLeaseActivation { + pub const fn is_activated(&self) -> bool { matches!(self, Self::Activated { .. }) } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum HlsAccessLeaseTouch { + Touched { lease: Box }, + Expired, + Denied, + UnknownLease, + SessionMismatch, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsAccessLeaseIdleRelease { + pub lease_id: HlsAccessLeaseId, + pub username: String, + pub user_session_token: String, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsAccessLeaseLifecycleSnapshot { + pub lease_id: HlsAccessLeaseId, + pub proxy_session_id: ProxySessionId, + pub state: HlsAccessLeaseState, + pub active_until_ms: Option, + pub pending_deadline: Option, + pub valid_until_ms: u64, + pub idle_release: Option, +} + +/// Registry for user-specific HLS access leases above shared content sessions. +#[derive(Debug, Default)] +pub struct HlsAccessLeaseStore { + by_lease_id: HashMap, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsAccessLeaseSessionSnapshot { + pub active_count: usize, + pub effective_origin_policy: Option, + pub idle_releases: Vec, +} + +impl HlsAccessLeaseStore { + pub fn prepare_access_lease(&mut self, lease: HlsAccessLease) { + self.by_lease_id.insert(lease.lease_id.clone(), lease); + } + + pub fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) -> Option { + self.by_lease_id.remove(lease_id) + } + + pub fn remove_access_leases_for_session(&mut self, proxy_session_id: &ProxySessionId) -> Vec { + let lease_ids = self + .by_lease_id + .values() + .filter(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.lease_id.clone()) + .collect::>(); + lease_ids + .into_iter() + .filter_map(|lease_id| self.by_lease_id.remove(&lease_id)) + .collect() + } + + pub fn clear(&mut self) -> usize { + let removed = self.by_lease_id.len(); + self.by_lease_id.clear(); + removed + } + + pub fn len(&self) -> usize { self.by_lease_id.len() } + + pub fn is_empty(&self) -> bool { self.by_lease_id.is_empty() } + + pub fn first_username_for_session(&self, proxy_session_id: &ProxySessionId) -> Option { + self.by_lease_id + .values() + .find(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.username.clone()) + } + + pub fn response_snapshot(&mut self, lease_id: &HlsAccessLeaseId, path_proxy_session_id: &ProxySessionId, now_ms: u64) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + lease.refresh_validity(now_ms); + Some(lease.clone()) + } + + pub fn mark_channel_unavailable_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> usize { + let mut marked = 0; + for lease in self.by_lease_id.values_mut() { + if lease.proxy_session_id != *proxy_session_id { + continue; + } + lease.refresh_validity(now_ms); + if lease_state_allows_use(lease.state) { + lease.response_flag = Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { + reason, + set_at_ms: now_ms, + }); + marked += 1; + } + } + marked + } + + pub fn mark_channel_unavailable_for_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> bool { + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return false; + }; + lease.refresh_validity(now_ms); + if !lease_state_allows_use(lease.state) { + return false; + } + lease.response_flag = Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { reason, set_at_ms: now_ms }); + true + } + + pub fn prune_expired_access_leases(&mut self, now_ms: u64) -> usize { + let initial_len = self.by_lease_id.len(); + self.by_lease_id.retain(|_, lease| { + lease.refresh_validity(now_ms); + lease.state != HlsAccessLeaseState::Expired + }); + initial_len.saturating_sub(self.by_lease_id.len()) + } + + pub fn access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let state = { + let lease = self.by_lease_id.get_mut(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + lease.refresh_validity(now_ms); + lease.state + }; + if state == HlsAccessLeaseState::Expired { + return None; + } + if state == HlsAccessLeaseState::Denied { + return self.by_lease_id.get(lease_id).cloned(); + } + if !lease_state_allows_use(state) { + return None; + } + self.by_lease_id.get(lease_id).cloned() + } + + pub fn update_origin_acquire_policy( + &mut self, + lease_id: &HlsAccessLeaseId, + connection_kind: ConnectionKind, + priority: i8, + ) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + if !lease_state_allows_use(lease.state) { + return None; + } + lease.update_origin_acquire_policy(connection_kind, priority); + Some(lease.clone()) + } + + pub fn activate_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> HlsAccessLeaseActivation { + let Some(new_lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseActivation::UnknownLease; + }; + if &new_lease.proxy_session_id != path_proxy_session_id { + return HlsAccessLeaseActivation::SessionMismatch; + } + if !lease_state_allows_use(new_lease.state) { + return activation_for_state(new_lease.state); + } + new_lease.refresh_validity(now_ms); + if new_lease.state == HlsAccessLeaseState::Expired { + return HlsAccessLeaseActivation::Expired; + } + + let previous_state = new_lease.state; + new_lease.state = HlsAccessLeaseState::Activated; + new_lease.last_seen_at_ms = now_ms; + new_lease.active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + new_lease.pending_deadline = None; + new_lease.valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + let lease = new_lease.clone(); + + HlsAccessLeaseActivation::Activated { lease: Box::new(lease), previous_state } + } + + pub fn touch_manifest_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + active_timing: Option, + pending_deadline: Option, + valid_window_ms: u64, + ) -> HlsAccessLeaseTouch { + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseTouch::UnknownLease; + }; + if &lease.proxy_session_id != path_proxy_session_id { + return HlsAccessLeaseTouch::SessionMismatch; + } + if !lease_state_allows_use(lease.state) { + return touch_for_state(lease.state); + } + lease.refresh_validity(now_ms); + if lease.state == HlsAccessLeaseState::Expired { + return HlsAccessLeaseTouch::Expired; + } + lease.last_seen_at_ms = now_ms; + match lease.state { + HlsAccessLeaseState::Pending => { + if let Some(pending_deadline) = pending_deadline { + lease.apply_pending_deadline(pending_deadline); + } + } + HlsAccessLeaseState::Activated => { + if let Some(timing) = active_timing { + lease.active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + lease.valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + } else { + lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); + } + } + HlsAccessLeaseState::Idle => { + lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); + } + HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied => {} + } + HlsAccessLeaseTouch::Touched { lease: Box::new(lease.clone()) } + } + + pub fn mark_pending_manifest_follow_up_for_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + deadline: HlsAccessLeasePendingDeadline, + ) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + lease.refresh_validity(now_ms); + if lease.state != HlsAccessLeaseState::Pending { + return None; + } + lease.last_seen_at_ms = now_ms; + if !lease.apply_pending_deadline(deadline) { + return None; + } + Some(lease.clone()) + } + + pub fn mark_pending_manifest_follow_up_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + deadline: HlsAccessLeasePendingDeadline, + ) -> Vec { + let mut leases = Vec::new(); + for lease in self.by_lease_id.values_mut() { + if lease.proxy_session_id != *proxy_session_id { + continue; + } + lease.refresh_validity(now_ms); + if lease.state != HlsAccessLeaseState::Pending { + continue; + } + lease.last_seen_at_ms = now_ms; + if lease.apply_pending_deadline(deadline) { + leases.push(lease.clone()); + } + } + leases + } + + pub fn touch_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> bool { + self.touch_access_lease_snapshot(lease_id, now_ms, timing).is_some() + } + + pub fn touch_access_lease_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + if lease.state != HlsAccessLeaseState::Activated { + return None; + } + lease.refresh_validity(now_ms); + if lease.state == HlsAccessLeaseState::Expired { + return None; + } + lease.last_seen_at_ms = now_ms; + lease.active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + lease.valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + Some(lease.clone()) + } + + pub fn deny_access_lease(&mut self, lease_id: &HlsAccessLeaseId) { + if let Some(lease) = self.by_lease_id.get_mut(lease_id) { + lease.state = HlsAccessLeaseState::Denied; + } + } + + pub fn lease_state(&self, lease_id: &HlsAccessLeaseId, now_ms: u64) -> Option { + self.by_lease_id.get(lease_id).map(|lease| { + if lease.validity_due_at_ms() <= now_ms { + HlsAccessLeaseState::Expired + } else { + lease.state + } + }) + } + + pub fn active_access_lease_count_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { + let mut active_count = 0; + for lease in self.by_lease_id.values_mut() { + if lease.proxy_session_id == *proxy_session_id { + lease.refresh_validity(now_ms); + if lease.state == HlsAccessLeaseState::Activated + && lease.active_until_ms.is_some_and(|active_until| active_until > now_ms) + { + active_count += 1; + } + } + } + active_count + } + + pub fn has_usable_access_lease_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { + let mut has_usable_lease = false; + for lease in self.by_lease_id.values_mut() { + lease.refresh_validity(now_ms); + if lease.proxy_session_id == *proxy_session_id + && (lease.state == HlsAccessLeaseState::Pending + || lease.state == HlsAccessLeaseState::Idle + || (lease.state == HlsAccessLeaseState::Activated && lease.valid_until_ms > now_ms)) + { + has_usable_lease = true; + } + } + has_usable_lease + } + + pub fn session_snapshot( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> HlsAccessLeaseSessionSnapshot { + let mut active_count = 0; + let mut effective_origin_policy = None; + let mut idle_releases = Vec::new(); + for lease in self.by_lease_id.values_mut() { + if lease.proxy_session_id != *proxy_session_id { + continue; + } + if let Some(release) = lease.refresh_activity(now_ms) { + idle_releases.push(release); + } + if lease.state == HlsAccessLeaseState::Activated { + active_count += 1; + } + if matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) { + let candidate = + HlsEffectiveOriginAcquirePolicy::new(lease.origin_connection_kind, lease.origin_priority, now_ms); + effective_origin_policy = Some(effective_origin_policy.map_or(candidate, |current| { + if candidate.is_better_than(current) { + candidate + } else { + current + } + })); + } + } + HlsAccessLeaseSessionSnapshot { active_count, effective_origin_policy, idle_releases } + } + + pub fn lifecycle_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + let idle_release = lease.refresh_activity(now_ms); + Some(HlsAccessLeaseLifecycleSnapshot { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + state: lease.state, + active_until_ms: lease.active_until_ms, + pending_deadline: lease.pending_deadline, + valid_until_ms: lease.valid_until_ms, + idle_release, + }) + } +} + +pub fn new_hls_access_lease_id() -> HlsAccessLeaseId { + let mut bytes = [0u8; HLS_ACCESS_LEASE_ID_BYTES]; + if OsRng.try_fill_bytes(&mut bytes).is_err() { + rand::rng().fill_bytes(&mut bytes); + } + HlsAccessLeaseId(general_purpose::URL_SAFE_NO_PAD.encode(bytes)) +} + +const fn lease_state_allows_use(state: HlsAccessLeaseState) -> bool { + matches!(state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) +} + +const fn activation_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseActivation { + match state { + HlsAccessLeaseState::Expired => HlsAccessLeaseActivation::Expired, + HlsAccessLeaseState::Denied => HlsAccessLeaseActivation::Denied, + HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { + HlsAccessLeaseActivation::UnknownLease + } + } +} + +const fn touch_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseTouch { + match state { + HlsAccessLeaseState::Expired => HlsAccessLeaseTouch::Expired, + HlsAccessLeaseState::Denied => HlsAccessLeaseTouch::Denied, + HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { + HlsAccessLeaseTouch::UnknownLease + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, + HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseResponseFlag, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsPlaybackFamilyKey, + }; + use crate::api::model::{ConnectionKind, ProxySessionId}; + use axum::http::StatusCode; + + fn lease(lease_id: HlsAccessLeaseId, proxy_session_id: &str, now_ms: u64) -> HlsAccessLease { + HlsAccessLease::pending( + lease_id, + HlsPlaybackFamilyKey::new("alice", "client-a"), + ProxySessionId(proxy_session_id.to_string()), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 15_000, + ) + } + + const fn timing(active_window_ms: u64, valid_window_ms: u64) -> HlsAccessLeaseTiming { + HlsAccessLeaseTiming { active_window_ms, valid_window_ms } + } + + #[test] + fn access_lease_id_is_short_and_opaque() { + let lease_id = new_hls_access_lease_id(); + + assert_eq!(lease_id.0.len(), 22); + assert!(!lease_id.0.contains("alice")); + assert!(!lease_id.0.contains("session")); + } + + #[test] + fn access_lease_activates_and_slides_validity() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store + .activate_access_lease(&lease_id, &proxy_session_id, 10_000, timing(5_000, 30_000)) + .is_activated()); + assert_eq!(store.lease_state(&lease_id, 24_999), Some(HlsAccessLeaseState::Activated)); + assert!(store.touch_access_lease(&lease_id, 24_000, timing(5_000, 30_000))); + assert_eq!(store.lease_state(&lease_id, 53_999), Some(HlsAccessLeaseState::Activated)); + } + + #[test] + fn access_lease_idles_at_exact_active_until_boundary() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + + let snapshot = store.lifecycle_snapshot(&lease_id, 7_000).expect("lease should exist"); + + assert_eq!(snapshot.state, HlsAccessLeaseState::Idle); + assert!(snapshot.idle_release.is_some()); + assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 7_000), 0); + } + + #[test] + fn access_lease_expires_at_exact_valid_until_boundary() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + let snapshot = store.lifecycle_snapshot(&lease_id, 17_000).expect("lease should exist"); + + assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); + assert!(snapshot.idle_release.is_some()); + assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); + } + + #[test] + fn access_lease_expires_without_activity() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert_eq!( + store.activate_access_lease(&lease_id, &proxy_session_id, 17_000, timing(5_000, 15_000)), + HlsAccessLeaseActivation::Expired + ); + } + + #[test] + fn channel_unavailable_flag_marks_only_usable_leases_for_session() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let other_proxy_session_id = ProxySessionId("other".to_string()); + let pending_lease_id = HlsAccessLeaseId("pending".to_string()); + let expired_lease_id = HlsAccessLeaseId("expired".to_string()); + let other_lease_id = HlsAccessLeaseId("other".to_string()); + store.prepare_access_lease(lease(pending_lease_id.clone(), &proxy_session_id.0, 5_000)); + store.prepare_access_lease(lease(expired_lease_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(other_lease_id.clone(), &other_proxy_session_id.0, 1_000)); + assert_eq!( + store.activate_access_lease(&expired_lease_id, &proxy_session_id, 17_000, timing(5_000, 15_000)), + HlsAccessLeaseActivation::Expired + ); + + let marked = store.mark_channel_unavailable_for_session( + &proxy_session_id, + 17_000, + HlsAccessLeaseChannelUnavailableReason::SegmentPermanentFailure { + status: Some(StatusCode::NOT_FOUND), + }, + ); + + assert_eq!(marked, 1); + let pending = store.response_snapshot(&pending_lease_id, &proxy_session_id, 17_000).unwrap(); + assert!(matches!( + pending.response_flag, + Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { + reason: HlsAccessLeaseChannelUnavailableReason::SegmentPermanentFailure { + status: Some(StatusCode::NOT_FOUND) + }, + .. + }) + )); + assert!(store + .response_snapshot(&other_lease_id, &other_proxy_session_id, 17_000) + .unwrap() + .response_flag + .is_none()); + } + + #[test] + fn channel_unavailable_flag_can_mark_single_lease() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let other_lease_id = HlsAccessLeaseId("lease-b".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(other_lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.mark_channel_unavailable_for_lease( + &lease_id, + 2_000, + HlsAccessLeaseChannelUnavailableReason::ResourceWaitThresholdExceeded, + )); + + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).unwrap(); + assert!(matches!( + lease.response_flag, + Some(HlsAccessLeaseResponseFlag::ChannelUnavailable { + reason: HlsAccessLeaseChannelUnavailableReason::ResourceWaitThresholdExceeded, + set_at_ms: 2_000 + }) + )); + assert!(store + .response_snapshot(&other_lease_id, &proxy_session_id, 2_000) + .unwrap() + .response_flag + .is_none()); + } + + #[test] + fn same_family_leases_remain_independently_valid() { + let mut store = HlsAccessLeaseStore::default(); + let old_lease_id = HlsAccessLeaseId("old".to_string()); + let new_lease_id = HlsAccessLeaseId("new".to_string()); + let proxy_a = ProxySessionId("proxy-a".to_string()); + let proxy_b = ProxySessionId("proxy-b".to_string()); + let family = HlsPlaybackFamilyKey::new("alice", "client-a"); + + store.prepare_access_lease(HlsAccessLease::pending( + old_lease_id.clone(), + family.clone(), + proxy_a.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + 1_000, + 15_000, + )); + assert!(store.activate_access_lease(&old_lease_id, &proxy_a, 2_000, timing(5_000, 15_000)).is_activated()); + store.prepare_access_lease(HlsAccessLease::pending( + new_lease_id.clone(), + family, + proxy_b.clone(), + "alice".to_string(), + "session-b".to_string(), + 1, + "67890".to_string(), + 67890, + 3_000, + 15_000, + )); + + let activation = store.activate_access_lease(&new_lease_id, &proxy_b, 4_000, timing(5_000, 15_000)); + assert!(activation.is_activated()); + assert_eq!(store.lease_state(&old_lease_id, 4_000), Some(HlsAccessLeaseState::Activated)); + assert!(store.touch_access_lease(&old_lease_id, 5_000, timing(5_000, 15_000))); + assert_eq!(store.lease_state(&old_lease_id, 19_999), Some(HlsAccessLeaseState::Activated)); + } + + #[test] + fn manifest_touch_extends_activated_lease_active_window() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 6_000, + Some(timing(10_000, 30_000)), + None, + 15_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert_eq!(lease.last_seen_at_ms, 6_000); + assert_eq!(lease.pending_deadline, None); + assert_eq!(lease.active_until_ms, Some(16_000)); + assert_eq!(lease.valid_until_ms, 36_000); + } + + #[test] + fn pending_lease_expires_at_pending_deadline_even_when_valid_window_is_longer() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let mut lease = lease(lease_id.clone(), &proxy_session_id.0, 1_000); + lease.pending_deadline = Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 6_000 }); + lease.valid_until_ms = 31_000; + store.prepare_access_lease(lease); + + assert_eq!(store.lease_state(&lease_id, 5_999), Some(HlsAccessLeaseState::Pending)); + let snapshot = store.lifecycle_snapshot(&lease_id, 6_000).expect("lease should exist"); + assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); + assert!(snapshot.idle_release.is_some(), "pending expiry must release counted user admission"); + assert_eq!(store.lease_state(&lease_id, 6_000), Some(HlsAccessLeaseState::Expired)); + } + + #[test] + fn manifest_touch_can_shorten_pending_lease_to_follow_up_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Pending); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + assert_eq!(store.lease_state(&lease_id, 11_999), Some(HlsAccessLeaseState::Pending)); + assert_eq!(store.lease_state(&lease_id, 12_000), Some(HlsAccessLeaseState::Expired)); + } + + #[test] + fn bootstrap_touch_cannot_extend_existing_follow_up_pending_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 3_000, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms: 100_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + } + + #[test] + fn repeated_follow_up_touch_cannot_extend_existing_pending_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 3_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + } + + #[test] + fn session_follow_up_shortens_pending_lease_once() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + let shortened = store.mark_pending_manifest_follow_up_for_session( + &proxy_session_id, + 2_000, + HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }, + ); + assert_eq!(shortened.len(), 1); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + + let unchanged = store.mark_pending_manifest_follow_up_for_session( + &proxy_session_id, + 3_000, + HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }, + ); + assert!(unchanged.is_empty()); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + } + + #[test] + fn activated_lease_remains_valid_after_media_touch() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + assert!(store.touch_access_lease(&lease_id, 3_000, timing(5_000, 15_000))); + assert_eq!(store.lease_state(&lease_id, 17_999), Some(HlsAccessLeaseState::Activated)); + } + + #[test] + fn activated_lease_becomes_idle_after_active_window_but_remains_reactivatable() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + + let snapshot = store.session_snapshot(&proxy_session_id, 8_000); + assert_eq!(snapshot.active_count, 0); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); + assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Idle)); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 8_000)); + assert!(store.access_lease(&lease_id, &proxy_session_id, 8_000).is_some()); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 8_000, timing(5_000, 30_000)).is_activated()); + assert_eq!(store.session_snapshot(&proxy_session_id, 8_000).active_count, 1); + } + + #[test] + fn activated_lease_validity_expiry_reports_idle_release() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(30_000, 5_000)).is_activated()); + + let snapshot = store.session_snapshot(&proxy_session_id, 8_000); + assert_eq!(snapshot.active_count, 0); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); + assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Expired)); + } + + #[test] + fn usable_access_lease_query_accepts_pending_idle_and_active_activated_only() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let pending_id = HlsAccessLeaseId("pending".to_string()); + let idle_id = HlsAccessLeaseId("idle".to_string()); + let activated_id = HlsAccessLeaseId("activated".to_string()); + let denied_id = HlsAccessLeaseId("denied".to_string()); + let expired_id = HlsAccessLeaseId("expired".to_string()); + + store.prepare_access_lease(lease(pending_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(idle_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(activated_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(denied_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(expired_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store + .activate_access_lease(&idle_id, &proxy_session_id, 2_000, timing(1_000, 15_000)) + .is_activated()); + assert!(store + .activate_access_lease(&activated_id, &proxy_session_id, 2_000, timing(5_000, 15_000)) + .is_activated()); + store.deny_access_lease(&denied_id); + + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); + let snapshot = store.session_snapshot(&proxy_session_id, 3_000); + assert_eq!(snapshot.active_count, 1); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, idle_id); + assert_eq!(store.lease_state(&idle_id, 3_000), Some(HlsAccessLeaseState::Idle)); + assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 3_000), 1); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); + + store.deny_access_lease(&pending_id); + store.deny_access_lease(&activated_id); + store.deny_access_lease(&expired_id); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); + store.deny_access_lease(&idle_id); + assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); + assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 17_000)); + assert_eq!(store.lease_state(&expired_id, 17_000), Some(HlsAccessLeaseState::Expired)); + } + + #[test] + fn session_snapshot_prefers_normal_origin_policy_over_soft() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease( + lease(HlsAccessLeaseId("soft".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Soft, -20), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("normal".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, 50), + ); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, 50); + } + + #[test] + fn origin_policy_update_reclassifies_existing_access_lease() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + store.prepare_access_lease( + lease(lease_id.clone(), &proxy_session_id.0, 1_000).with_origin_acquire_policy(ConnectionKind::Soft, 20), + ); + + let updated = + store.update_origin_acquire_policy(&lease_id, ConnectionKind::Normal, -5).expect("lease should update"); + assert_eq!(updated.origin_connection_kind, ConnectionKind::Normal); + assert_eq!(updated.origin_priority, -5); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("updated policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, -5); + } + + #[test] + fn session_snapshot_uses_best_priority_within_same_origin_policy_kind() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease( + lease(HlsAccessLeaseId("low-priority".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, 30), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("high-priority".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, -5), + ); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, -5); + } + + #[test] + fn session_snapshot_ignores_expired_and_denied_origin_policies() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let denied_id = HlsAccessLeaseId("denied".to_string()); + store.prepare_access_lease( + lease(denied_id.clone(), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, -100), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("expired".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, -50), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("active-soft".to_string()), &proxy_session_id.0, 10_000) + .with_origin_acquire_policy(ConnectionKind::Soft, 10), + ); + store.deny_access_lease(&denied_id); + + let snapshot = store.session_snapshot(&proxy_session_id, 17_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Soft); + assert_eq!(policy.priority, 10); + } + + #[test] + fn expired_lease_lookup_rejects_stale_entry_without_removing_before_lifecycle() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.access_lease(&lease_id, &proxy_session_id, 17_000).is_none()); + assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); + } +} diff --git a/backend/src/api/model/hls_cache/lifecycle.rs b/backend/src/api/model/hls_cache/lifecycle.rs new file mode 100644 index 000000000..a9dffce22 --- /dev/null +++ b/backend/src/api/model/hls_cache/lifecycle.rs @@ -0,0 +1,175 @@ +use super::{HlsAccessLeaseId, ProxySessionId}; +use std::{ + cmp::Ordering, + collections::{BinaryHeap, HashMap}, + time::Duration, +}; +use tokio::sync::{Mutex, Notify}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum HlsLifecycleEventKey { + AccessLeaseActive { lease_id: HlsAccessLeaseId, proxy_session_id: ProxySessionId }, + AccessLeaseValidity { lease_id: HlsAccessLeaseId, proxy_session_id: ProxySessionId }, + SessionIdle { proxy_session_id: ProxySessionId }, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsLifecycleEvent { + pub key: HlsLifecycleEventKey, + pub due_at_ms: u64, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +struct ScheduledInfo { + due_at_ms: u64, + sequence: u64, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct QueueEntry { + key: HlsLifecycleEventKey, + due_at_ms: u64, + sequence: u64, +} + +impl Ord for QueueEntry { + fn cmp(&self, other: &Self) -> Ordering { + other.due_at_ms.cmp(&self.due_at_ms).then_with(|| other.sequence.cmp(&self.sequence)) + } +} + +impl PartialOrd for QueueEntry { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } +} + +#[derive(Debug, Default)] +struct HlsLifecycleState { + scheduled: HashMap, + queue: BinaryHeap, + next_sequence: u64, +} + +#[derive(Debug, Default)] +pub struct HlsLifecycleManager { + state: Mutex, + notify: Notify, +} + +impl HlsLifecycleManager { + pub fn new() -> Self { Self::default() } + + pub async fn schedule(&self, key: HlsLifecycleEventKey, due_at_ms: u64) { + let should_notify = { + let mut state = self.state.lock().await; + state.next_sequence = state.next_sequence.saturating_add(1); + let sequence = state.next_sequence; + let previous_earliest = state.queue.peek().map(|entry| entry.due_at_ms); + state.scheduled.insert(key.clone(), ScheduledInfo { due_at_ms, sequence }); + state.queue.push(QueueEntry { key, due_at_ms, sequence }); + previous_earliest.is_none_or(|previous| due_at_ms <= previous) + }; + if should_notify { + self.notify.notify_one(); + } + } + + pub async fn cancel(&self, key: &HlsLifecycleEventKey) { + self.state.lock().await.scheduled.remove(key); + self.notify.notify_one(); + } + + pub async fn next_event(&self, cancel_token: &CancellationToken) -> Option { + loop { + let wait = { + let mut state = self.state.lock().await; + let now_ms = current_time_millis(); + loop { + let Some(entry) = state.queue.peek() else { + break LifecycleWait::Notify; + }; + let Some(info) = state.scheduled.get(&entry.key) else { + state.queue.pop(); + continue; + }; + if info.due_at_ms != entry.due_at_ms || info.sequence != entry.sequence { + state.queue.pop(); + continue; + } + if entry.due_at_ms <= now_ms { + let Some(entry) = state.queue.pop() else { + break LifecycleWait::Notify; + }; + state.scheduled.remove(&entry.key); + break LifecycleWait::Ready(HlsLifecycleEvent { key: entry.key, due_at_ms: entry.due_at_ms }); + } + break LifecycleWait::Sleep(Duration::from_millis(entry.due_at_ms.saturating_sub(now_ms))); + } + }; + + match wait { + LifecycleWait::Ready(event) => return Some(event), + LifecycleWait::Sleep(duration) => { + tokio::select! { + () = cancel_token.cancelled() => return None, + () = self.notify.notified() => {} + () = tokio::time::sleep(duration) => {} + } + } + LifecycleWait::Notify => { + tokio::select! { + () = cancel_token.cancelled() => return None, + () = self.notify.notified() => {} + } + } + } + } + } + + #[cfg(test)] + pub async fn scheduled_len(&self) -> usize { self.state.lock().await.scheduled.len() } +} + +enum LifecycleWait { + Ready(HlsLifecycleEvent), + Sleep(Duration), + Notify, +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{HlsLifecycleEventKey, HlsLifecycleManager}; + use crate::api::model::{HlsAccessLeaseId, ProxySessionId}; + use tokio_util::sync::CancellationToken; + + #[tokio::test] + async fn rescheduling_same_key_discards_stale_event() { + let lifecycle = HlsLifecycleManager::new(); + let cancel = CancellationToken::new(); + let key = HlsLifecycleEventKey::AccessLeaseActive { + lease_id: HlsAccessLeaseId("lease".to_string()), + proxy_session_id: ProxySessionId("proxy".to_string()), + }; + + lifecycle.schedule(key.clone(), 1).await; + lifecycle.schedule(key.clone(), 2).await; + + let event = lifecycle.next_event(&cancel).await.expect("event should fire"); + assert_eq!(event.key, key); + assert_eq!(event.due_at_ms, 2); + assert_eq!(lifecycle.scheduled_len().await, 0); + } + + #[tokio::test] + async fn cancel_removes_pending_key() { + let lifecycle = HlsLifecycleManager::new(); + let key = HlsLifecycleEventKey::SessionIdle { proxy_session_id: ProxySessionId("proxy".to_string()) }; + + lifecycle.schedule(key.clone(), u64::MAX).await; + lifecycle.cancel(&key).await; + + assert_eq!(lifecycle.scheduled_len().await, 0); + } +} diff --git a/backend/src/api/model/hls_cache/manager.rs b/backend/src/api/model/hls_cache/manager.rs new file mode 100644 index 000000000..19fbc1f7a --- /dev/null +++ b/backend/src/api/model/hls_cache/manager.rs @@ -0,0 +1,1259 @@ +use super::{ + build_rewrite_secret_fingerprint, safe_proxy_session_id, safe_session_key, GarbageCollectionPolicy, + HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, + HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseSessionSnapshot, + HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsCacheMetrics, + HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsGarbageCollector, HlsLifecycleEvent, HlsLifecycleEventKey, + HlsLifecycleManager, HlsMapWorkerPool, + HlsOriginSource, HlsQosRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, + HlsSessionKey, HlsSessionStore, HlsSessionStoreOutcome, ProxySessionId, SegmentFetchPolicy, + TransientResourceStore, +}; +use crate::{ + api::model::{ActiveProviderManager, ActiveUserManager, AppState}, + model::{AppConfig, HlsCacheConfig, StripConfig}, +}; +use crate::model::HlsManifestRecoveryBurstConfig; +use arc_swap::ArcSwap; +use log::{debug, error, info}; +use shared::utils::sanitize_sensitive_info; +use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; +use tokio::sync::{RwLock, Semaphore}; +use tokio_util::sync::CancellationToken; + +/// Root runtime object for the future HLS cache proxy. +pub struct HlsProxyManager { + sessions: Arc, + segment_cache: Arc, + segment_repair: Arc, + segment_worker_pool: Arc, + map_worker_pool: Arc, + runtime_config: ArcSwap, + transient_resources: Arc, + access_leases: Arc>, + lifecycle: Arc, + account_overlap_cooldowns: Arc>>, + metrics: Arc, + qos: Arc, + gc: Arc, +} + +#[derive(Debug, Clone)] +struct HlsProxyRuntimeConfig { + enabled: bool, + segment_fetch_policy: SegmentFetchPolicy, + cache_duration_seconds: u64, + strip: StripConfig, + origin_manifest_timeout_ms: u64, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig, + transient_resource_ttl_ms: u64, + gc_policy: GarbageCollectionPolicy, + rewrite_secret_fingerprint: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct HlsAccountOverlapCooldownKey { + input_name: Arc, + account_name: Arc, +} + +#[derive(Debug, Clone, Copy)] +struct HlsAccountOverlapCooldown { + until_ms: u64, +} + +#[derive(Debug, Clone, Copy)] +enum HlsAccountOverlapCooldownReason { + ReclaimedByOriginalOwner, + SpeculativePromoted, +} + +impl HlsAccountOverlapCooldownReason { + fn as_log_reason(self) -> &'static str { + match self { + Self::ReclaimedByOriginalOwner => "reclaimed-by-original-owner", + Self::SpeculativePromoted => "speculative-promoted", + } + } +} + +fn hls_pending_manifest_follow_up_window_ms(target_duration: Option) -> u64 { + let target_duration_secs = u64::from(target_duration.unwrap_or(15)).max(1); + target_duration_secs.saturating_mul(2_000).max(10_000) +} + +fn hls_pending_manifest_follow_up_deadline( + now_ms: u64, + target_duration: Option, +) -> HlsAccessLeasePendingDeadline { + HlsAccessLeasePendingDeadline::FollowUp { + deadline_ms: now_ms.saturating_add(hls_pending_manifest_follow_up_window_ms(target_duration)), + } +} + +impl HlsProxyRuntimeConfig { + fn from_config(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { + Self::from_config_with_enabled(config, rewrite_secret, true) + } + + fn from_config_with_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { + Self { + enabled, + segment_fetch_policy: SegmentFetchPolicy::from_config(config), + cache_duration_seconds: config.cache_duration, + strip: config.strip.clone(), + origin_manifest_timeout_ms: config.origin_manifest_timeout_ms, + manifest_recovery_burst: config.manifest_recovery_burst.clone(), + transient_resource_ttl_ms: config.cache_duration.saturating_mul(1_000), + gc_policy: GarbageCollectionPolicy::from_config(config), + rewrite_secret_fingerprint: build_rewrite_secret_fingerprint(rewrite_secret), + } + } +} + +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +struct HlsProxySessionCleanupStats { + access_leases: usize, + repair_windows: usize, + repair_generations: usize, + repair_candidates: usize, + repair_object_metadata: usize, + repair_watchdog_metadata: usize, + repair_watchdog_locks: usize, + qos_access_leases: usize, +} + +impl HlsProxySessionCleanupStats { + fn did_cleanup(self) -> bool { + self.access_leases > 0 + || self.repair_windows > 0 + || self.repair_generations > 0 + || self.repair_candidates > 0 + || self.repair_object_metadata > 0 + || self.repair_watchdog_metadata > 0 + || self.repair_watchdog_locks > 0 + || self.qos_access_leases > 0 + } +} + +impl HlsProxyManager { + pub fn new() -> Self { + let default_dto = shared::model::HlsCacheConfigDto::default(); + let default_config = HlsCacheConfig::from(&default_dto); + Self::with_hls_cache_config(&default_config) + } + + pub fn from_hls_cache_config(config: Option<&HlsCacheConfig>) -> Self { + Self::from_hls_cache_config_and_secret(config, &[]) + } + + pub fn from_hls_cache_config_and_secret(config: Option<&HlsCacheConfig>, rewrite_secret: &[u8]) -> Self { + let default_config; + let (config, enabled) = if let Some(config) = config { + (config, true) + } else { + default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); + (&default_config, false) + }; + Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, enabled) + } + + pub fn with_cache_settings(cache_path: impl Into, cache_duration_seconds: u64) -> Self { + let default_dto = shared::model::HlsCacheConfigDto { + cache_duration: cache_duration_seconds, + cache_path: Some(cache_path.into().to_string_lossy().to_string()), + ..Default::default() + }; + let default_config = HlsCacheConfig::from(&default_dto); + let segment_fetch_policy = SegmentFetchPolicy::from_config(&default_config); + let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); + let sessions = Arc::new(HlsSessionStore::new()); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&default_config.cache_path))); + segment_cache.update_cache_limits(default_config.cache_bytes, default_config.cache_bytes_per_session); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(default_config.segment_repair.clone())); + let metrics = Arc::new(HlsCacheMetrics::default()); + let qos = Arc::new(HlsQosRegistry::default()); + let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); + let lifecycle = Arc::new(HlsLifecycleManager::new()); + let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); + let gc_policy = GarbageCollectionPolicy::from_config(&default_config); + let runtime_config = HlsProxyRuntimeConfig::from_config(&default_config, &[]); + let gc = Arc::new(HlsGarbageCollector::new_with_metrics( + Arc::clone(&sessions), + Arc::clone(&segment_cache), + gc_policy.clone(), + runtime_config.rewrite_secret_fingerprint.clone(), + Arc::clone(&metrics), + )); + Self { + sessions, + segment_cache, + segment_repair, + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_and_metrics( + segment_fetch_policy.clone(), + Arc::clone(&global_fetch_semaphore), + Arc::clone(&access_leases), + Arc::clone(&metrics), + )), + map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_and_access_leases( + segment_fetch_policy.clone(), + global_fetch_semaphore, + Arc::clone(&access_leases), + )), + runtime_config: ArcSwap::from_pointee(runtime_config), + transient_resources: Arc::new(TransientResourceStore::new()), + access_leases, + lifecycle, + account_overlap_cooldowns, + metrics, + qos, + gc, + } + } + + pub fn with_hls_cache_config(config: &HlsCacheConfig) -> Self { + Self::with_hls_cache_config_and_secret(config, &[]) + } + + pub fn with_hls_cache_config_and_secret(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { + Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, true) + } + + fn with_hls_cache_config_and_secret_enabled( + config: &HlsCacheConfig, + rewrite_secret: &[u8], + enabled: bool, + ) -> Self { + let segment_fetch_policy = SegmentFetchPolicy::from_config(config); + let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); + let sessions = Arc::new(HlsSessionStore::new()); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&config.cache_path))); + segment_cache.update_cache_limits(config.cache_bytes, config.cache_bytes_per_session); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(config.segment_repair.clone())); + let metrics = Arc::new(HlsCacheMetrics::default()); + let qos = Arc::new(HlsQosRegistry::default()); + let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); + let lifecycle = Arc::new(HlsLifecycleManager::new()); + let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); + let gc_policy = GarbageCollectionPolicy::from_config(config); + let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(config, rewrite_secret, enabled); + let gc = Arc::new(HlsGarbageCollector::new_with_metrics( + Arc::clone(&sessions), + Arc::clone(&segment_cache), + gc_policy.clone(), + runtime_config.rewrite_secret_fingerprint.clone(), + Arc::clone(&metrics), + )); + Self { + sessions, + segment_cache, + segment_repair, + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_and_metrics( + segment_fetch_policy.clone(), + Arc::clone(&global_fetch_semaphore), + Arc::clone(&access_leases), + Arc::clone(&metrics), + )), + map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_and_access_leases( + segment_fetch_policy.clone(), + global_fetch_semaphore, + Arc::clone(&access_leases), + )), + runtime_config: ArcSwap::from_pointee(runtime_config), + transient_resources: Arc::new(TransientResourceStore::new()), + access_leases, + lifecycle, + account_overlap_cooldowns, + metrics, + qos, + gc, + } + } + + pub fn sessions(&self) -> &Arc { &self.sessions } + + pub fn segment_cache(&self) -> &Arc { &self.segment_cache } + + pub fn segment_repair(&self) -> &Arc { &self.segment_repair } + + pub fn segment_worker_pool(&self) -> &Arc { &self.segment_worker_pool } + + pub fn map_worker_pool(&self) -> &Arc { &self.map_worker_pool } + + pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() } + + pub fn is_enabled(&self) -> bool { self.runtime_config.load().enabled } + + pub fn cache_duration_seconds(&self) -> u64 { self.runtime_config.load().cache_duration_seconds } + + pub fn session_idle_timeout_ms(&self) -> u64 { self.runtime_config.load().gc_policy.session_idle_timeout_ms } + + pub fn strip(&self) -> StripConfig { self.runtime_config.load().strip.clone() } + + pub fn origin_manifest_timeout_ms(&self) -> u64 { self.runtime_config.load().origin_manifest_timeout_ms } + + pub fn manifest_recovery_burst(&self) -> HlsManifestRecoveryBurstConfig { + self.runtime_config.load().manifest_recovery_burst.clone() + } + + pub fn transient_resource_ttl_ms(&self) -> u64 { self.runtime_config.load().transient_resource_ttl_ms } + + pub fn transient_resources(&self) -> &Arc { &self.transient_resources } + + pub fn access_leases(&self) -> &Arc> { &self.access_leases } + + pub fn lifecycle(&self) -> &Arc { &self.lifecycle } + + pub fn metrics(&self) -> &Arc { &self.metrics } + + pub fn qos(&self) -> &Arc { &self.qos } + + pub fn garbage_collector(&self) -> &Arc { &self.gc } + + pub fn gc_policy(&self) -> GarbageCollectionPolicy { self.runtime_config.load().gc_policy.clone() } + + pub fn rewrite_secret_fingerprint(&self) -> String { self.runtime_config.load().rewrite_secret_fingerprint.clone() } + + pub async fn is_account_overlap_cooling_down( + &self, + input_name: &Arc, + account_name: &Arc, + now_ms: u64, + ) -> bool { + let key = HlsAccountOverlapCooldownKey { + input_name: Arc::clone(input_name), + account_name: Arc::clone(account_name), + }; + let mut cooldowns = self.account_overlap_cooldowns.write().await; + let Some(cooldown) = cooldowns.get(&key).copied() else { + return false; + }; + if now_ms >= cooldown.until_ms { + cooldowns.remove(&key); + return false; + } + true + } + + pub async fn mark_account_overlap_reclaimed_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + ) { + self.mark_account_overlap_cooldown( + input_name, + account_name, + now_ms, + hard_active_window_ms, + HlsAccountOverlapCooldownReason::ReclaimedByOriginalOwner, + ) + .await; + } + + pub async fn mark_account_overlap_promoted_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + ) { + self.mark_account_overlap_cooldown( + input_name, + account_name, + now_ms, + hard_active_window_ms, + HlsAccountOverlapCooldownReason::SpeculativePromoted, + ) + .await; + } + + async fn mark_account_overlap_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + reason: HlsAccountOverlapCooldownReason, + ) { + let until_ms = now_ms.saturating_add(hard_active_window_ms); + if until_ms <= now_ms { + return; + } + let key = HlsAccountOverlapCooldownKey { input_name, account_name }; + self.account_overlap_cooldowns + .write() + .await + .insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); + debug!( + "HLS account overlap cooldown set for input {} account {} until {} ms after {}", + sanitize_sensitive_info(key.input_name.as_ref()), + sanitize_sensitive_info(key.account_name.as_ref()), + until_ms, + reason.as_log_reason() + ); + } + + pub async fn update_config(&self, app_config: &AppConfig) { + let (hls_config, rewrite_secret, enabled) = { + let config = app_config.config.load(); + let rewrite_secret = config + .reverse_proxy + .as_ref() + .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); + let hls_config = config + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()) + .cloned(); + let enabled = hls_config.is_some(); + let hls_config = + hls_config.unwrap_or_else(|| HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default())); + (hls_config, rewrite_secret, enabled) + }; + let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(&hls_config, &rewrite_secret, enabled); + let cache_path_changed = self.segment_cache.update_cache_path(PathBuf::from(&hls_config.cache_path)); + self.segment_cache.update_cache_limits(hls_config.cache_bytes, hls_config.cache_bytes_per_session); + if cache_path_changed { + self.clear_runtime_cache_state_for_cache_path_change().await; + } + for session in self.sessions.list_sessions().await { + session + .write() + .await + .configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); + } + self.segment_repair.update_config(hls_config.segment_repair.clone()); + let global_fetch_semaphore = + Arc::new(Semaphore::new(runtime_config.segment_fetch_policy.max_global_segment_fetches)); + self.segment_worker_pool + .update_config(runtime_config.segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore)); + self.map_worker_pool.update_config(runtime_config.segment_fetch_policy.clone(), global_fetch_semaphore); + self.gc.update_config(runtime_config.gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone()); + self.runtime_config.store(Arc::new(runtime_config)); + } + + async fn clear_runtime_cache_state_for_cache_path_change(&self) { + self.sessions.clear().await; + let removed_leases = self.access_leases.write().await.clear(); + let removed_qos = self.qos.clear().await; + self.segment_repair.clear_runtime_state().await; + debug!( + "HLS cache runtime state cleared after cache path change: access_leases_removed={removed_leases} qos_access_leases_removed={removed_qos}" + ); + } + + pub async fn prepare_access_lease(&self, lease: HlsAccessLease) { + self.access_leases.write().await.prepare_access_lease(lease.clone()); + self.schedule_access_lease_validity(&lease).await; + } + + pub async fn access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let (lease, still_stored) = { + let mut access_leases = self.access_leases.write().await; + let lease = access_leases.access_lease(lease_id, proxy_session_id, now_ms); + let still_stored = access_leases.lease_state(lease_id, now_ms).is_some(); + (lease, still_stored) + }; + if lease.is_none() && !still_stored { + self.segment_repair.remove_access_lease_window(lease_id).await; + self.qos.remove_access_lease(lease_id).await; + } + lease + } + + pub async fn access_lease_response_snapshot( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.access_leases.write().await.response_snapshot(lease_id, proxy_session_id, now_ms) + } + + pub async fn mark_access_leases_channel_unavailable_for_session( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> usize { + self.access_leases + .write() + .await + .mark_channel_unavailable_for_session(proxy_session_id, now_ms, reason) + } + + pub async fn mark_access_lease_channel_unavailable( + &self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> bool { + self.access_leases.write().await.mark_channel_unavailable_for_lease(lease_id, now_ms, reason) + } + + pub async fn update_access_lease_origin_acquire_policy( + &self, + lease_id: &HlsAccessLeaseId, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + ) -> Option { + self.access_leases.write().await.update_origin_acquire_policy(lease_id, connection_kind, priority) + } + + pub async fn activate_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> HlsAccessLeaseActivation { + let activation = + self.access_leases.write().await.activate_access_lease(lease_id, proxy_session_id, now_ms, timing); + if let HlsAccessLeaseActivation::Activated { lease, previous_state } = &activation { + if matches!(previous_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle) { + self.segment_repair.start_access_lease_window(lease.lease_id.clone()).await; + } + self.schedule_access_lease_activity(lease).await; + self.schedule_access_lease_validity(lease).await; + } + activation + } + + pub async fn touch_manifest_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + active_timing: Option, + pending_deadline: Option, + ttl_ms: u64, + ) -> HlsAccessLeaseTouch { + let touch = self.access_leases.write().await.touch_manifest_access_lease( + lease_id, + proxy_session_id, + now_ms, + active_timing, + pending_deadline, + ttl_ms, + ); + if let HlsAccessLeaseTouch::Touched { lease } = &touch { + if lease.state == HlsAccessLeaseState::Activated { + self.schedule_access_lease_activity(lease).await; + } + self.schedule_access_lease_validity(lease).await; + } + touch + } + + pub async fn mark_pending_manifest_follow_up_for_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + target_duration: Option, + ) -> bool { + let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); + let lease = self + .access_leases + .write() + .await + .mark_pending_manifest_follow_up_for_lease(lease_id, proxy_session_id, now_ms, deadline); + if let Some(lease) = lease { + self.schedule_access_lease_validity(&lease).await; + debug!( + "HLS pending manifest lease shortened after manifest response: lease={} proxy_session={}", + super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id) + ); + true + } else { + false + } + } + + pub async fn mark_pending_manifest_follow_up_for_session( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + target_duration: Option, + ) -> usize { + let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); + let leases = self + .access_leases + .write() + .await + .mark_pending_manifest_follow_up_for_session(proxy_session_id, now_ms, deadline); + for lease in &leases { + self.schedule_access_lease_validity(lease).await; + } + leases.len() + } + + pub async fn touch_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> bool { + let lease = self.access_leases.write().await.touch_access_lease_snapshot(lease_id, now_ms, timing); + if let Some(lease) = lease { + self.schedule_access_lease_activity(&lease).await; + self.schedule_access_lease_validity(&lease).await; + true + } else { + false + } + } + + pub async fn active_access_lease_count_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { + self.access_leases.write().await.active_access_lease_count_for_session(proxy_session_id, now_ms) + } + + pub async fn has_usable_access_lease_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { + self.access_leases.write().await.has_usable_access_lease_for_session(proxy_session_id, now_ms) + } + + pub async fn access_lease_session_snapshot( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> HlsAccessLeaseSessionSnapshot { + self.access_leases.write().await.session_snapshot(proxy_session_id, now_ms) + } + + async fn access_lease_lifecycle_snapshot( + &self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Option { + self.access_leases.write().await.lifecycle_snapshot(lease_id, now_ms) + } + + async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) { + self.access_leases.write().await.remove_access_lease(lease_id); + self.segment_repair.remove_access_lease_window(lease_id).await; + self.qos.remove_access_lease(lease_id).await; + } + + async fn cleanup_proxy_session_state( + &self, + proxy_session_id: &ProxySessionId, + reason: &'static str, + ) -> HlsProxySessionCleanupStats { + let before = self.segment_repair.stats().await; + let removed_leases = self.access_leases.write().await.remove_access_leases_for_session(proxy_session_id); + let username = removed_leases.first().map(|lease| lease.username.clone()); + self.sessions.update_expired_session_marker_username(proxy_session_id, username).await; + let removed_lease_ids = removed_leases.iter().map(|lease| lease.lease_id.clone()).collect::>(); + self.segment_repair.remove_proxy_session_state(proxy_session_id, &removed_lease_ids).await; + let removed_qos = self.qos.remove_access_leases(&removed_lease_ids).await; + let removed_qos = removed_qos.saturating_add(self.qos.remove_proxy_session_state(proxy_session_id).await); + let after = self.segment_repair.stats().await; + let stats = HlsProxySessionCleanupStats { + access_leases: removed_lease_ids.len(), + repair_windows: before.windows.saturating_sub(after.windows), + repair_generations: before.generations.saturating_sub(after.generations), + repair_candidates: before.checked_candidates.saturating_sub(after.checked_candidates), + repair_object_metadata: before.object_metadata.saturating_sub(after.object_metadata), + repair_watchdog_metadata: before.watchdog_metadata.saturating_sub(after.watchdog_metadata), + repair_watchdog_locks: before.watchdog_locks.saturating_sub(after.watchdog_locks), + qos_access_leases: removed_qos, + }; + if stats.did_cleanup() { + debug!( + "HLS proxy session state cleaned: session={} reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_object_metadata={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", + safe_proxy_session_id(proxy_session_id), + reason, + stats.access_leases, + stats.repair_windows, + stats.repair_generations, + stats.repair_candidates, + stats.repair_object_metadata, + stats.repair_watchdog_metadata, + stats.repair_watchdog_locks, + stats.qos_access_leases + ); + } + stats + } + + pub async fn expired_session_marker( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.sessions + .expired_session_marker( + proxy_session_id, + now_ms, + self.session_idle_timeout_ms().saturating_mul(2).max(1), + ) + .await + } + + async fn cleanup_all_runtime_state(&self, reason: &'static str) { + let removed_access_leases = self.access_leases.write().await.clear(); + self.account_overlap_cooldowns.write().await.clear(); + let removed_qos = self.qos.clear().await; + let before = self.segment_repair.stats().await; + self.segment_repair.clear_runtime_state().await; + if removed_access_leases > 0 + || before.windows > 0 + || before.generations > 0 + || before.checked_candidates > 0 + || before.metadata > 0 + || before.object_metadata > 0 + || before.locks > 0 + || before.watchdog_metadata > 0 + || before.watchdog_locks > 0 + || removed_qos > 0 + { + debug!( + "HLS runtime state cleaned: reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", + reason, + removed_access_leases, + before.windows, + before.generations, + before.checked_candidates, + before.metadata, + before.object_metadata, + before.locks, + before.watchdog_metadata, + before.watchdog_locks, + removed_qos + ); + } + } + + async fn cleanup_after_garbage_collection(&self, report: &super::GarbageCollectionReport) { + if report.secret_cache_invalidated { + self.cleanup_all_runtime_state("secret-cache-invalidated").await; + return; + } + for proxy_session_id in &report.removed_session_ids { + self.cleanup_proxy_session_state(proxy_session_id, "gc-session-removed").await; + } + } + + async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) { + if let Some(active_until_ms) = lease.active_until_ms { + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseActive { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + }, + active_until_ms, + ) + .await; + } + } + + async fn schedule_access_lease_validity(&self, lease: &HlsAccessLease) { + let due_at_ms = if lease.state == HlsAccessLeaseState::Pending { + lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) + } else { + lease.valid_until_ms + }; + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + }, + due_at_ms, + ) + .await; + } + + async fn schedule_access_lease_lifecycle_snapshot(&self, snapshot: &HlsAccessLeaseLifecycleSnapshot) { + if snapshot.state == HlsAccessLeaseState::Activated { + if let Some(active_until_ms) = snapshot.active_until_ms { + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseActive { + lease_id: snapshot.lease_id.clone(), + proxy_session_id: snapshot.proxy_session_id.clone(), + }, + active_until_ms, + ) + .await; + } + } + if snapshot.state != HlsAccessLeaseState::Expired && snapshot.state != HlsAccessLeaseState::Denied { + let due_at_ms = if snapshot.state == HlsAccessLeaseState::Pending { + snapshot + .pending_deadline + .map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) + } else { + snapshot.valid_until_ms + }; + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: snapshot.lease_id.clone(), + proxy_session_id: snapshot.proxy_session_id.clone(), + }, + due_at_ms, + ) + .await; + } + } + + pub async fn schedule_session_idle_for_handle(&self, session: &HlsSessionHandle) { + let session_idle_timeout_ms = self.session_idle_timeout_ms(); + let (proxy_session_id, due_at_ms) = { + let session = session.read().await; + (session.proxy_session_id.clone(), session.idle_expiry_due_at_ms(session_idle_timeout_ms)) + }; + self.lifecycle.schedule(HlsLifecycleEventKey::SessionIdle { proxy_session_id }, due_at_ms).await; + } + + pub async fn mark_authorized_media_access_for_session(&self, session: &HlsSessionHandle, now_ms: u64) { + session.write().await.mark_authorized_media_access(now_ms); + self.schedule_session_idle_for_handle(session).await; + } + + pub async fn handle_lifecycle_event( + &self, + active_users: &Arc, + active_provider: &Arc, + event: HlsLifecycleEvent, + now_ms: u64, + ) { + if !self.is_enabled() { + return; + } + match event.key { + HlsLifecycleEventKey::AccessLeaseActive { lease_id, proxy_session_id } + | HlsLifecycleEventKey::AccessLeaseValidity { lease_id, proxy_session_id } => { + let mut should_sync_session = false; + if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await { + should_sync_session = true; + if let Some(release) = &snapshot.idle_release { + active_users + .release_session_streams_and_counted_reservation( + &release.username, + &release.user_session_token, + ) + .await; + debug!( + "HLS access lease idled: lease={} proxy_session={} session={}", + super::safe_hls_access_lease_id(&release.lease_id), + safe_proxy_session_id(&snapshot.proxy_session_id), + super::safe_user_session_token(&release.user_session_token) + ); + } + if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) { + self.remove_access_lease(&snapshot.lease_id).await; + debug!( + "HLS access lease removed: lease={} proxy_session={} state={}", + super::safe_hls_access_lease_id(&snapshot.lease_id), + safe_proxy_session_id(&snapshot.proxy_session_id), + snapshot.state.as_log_value() + ); + debug!( + "HLS lifecycle state snapshot: trigger=access-lease-removed {}", + self.debug_state_summary().await + ); + } else { + self.schedule_access_lease_lifecycle_snapshot(&snapshot).await; + } + } + if should_sync_session { + if let Some(session) = self.sessions.get_by_proxy_session_id(&proxy_session_id).await { + self.sync_session_access_lease_count_and_detach_if_needed( + active_users, + active_provider, + &session, + &proxy_session_id, + now_ms, + ) + .await; + } + } + } + HlsLifecycleEventKey::SessionIdle { proxy_session_id } => { + self.handle_session_idle_lifecycle_event(&proxy_session_id, now_ms).await; + } + } + } + + async fn handle_session_idle_lifecycle_event(&self, proxy_session_id: &ProxySessionId, now_ms: u64) { + let Some(session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { + return; + }; + let session_idle_timeout_ms = self.session_idle_timeout_ms(); + let (key, due_at_ms, can_remove) = { + let session = session.read().await; + ( + session.key.clone(), + session.idle_expiry_due_at_ms(session_idle_timeout_ms), + session.can_expire_idle_session(now_ms, session_idle_timeout_ms), + ) + }; + if !can_remove { + self.lifecycle + .schedule( + HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + due_at_ms.max(now_ms.saturating_add(1)), + ) + .await; + return; + } + if self.segment_cache.has_active_temp_files_for_session(proxy_session_id).await { + self.lifecycle + .schedule( + HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + now_ms.saturating_add(1_000), + ) + .await; + return; + } + let username = self.access_leases.read().await.first_username_for_session(proxy_session_id); + if self + .sessions + .remove_session_marking_expired( + &key, + proxy_session_id, + now_ms, + HlsExpiredSessionReason::SessionIdleTimeout, + username, + ) + .await + .is_some() + { + self.cleanup_proxy_session_state(proxy_session_id, "lifecycle-session-expired").await; + if let Err(err) = self.segment_cache.delete_session_dir(proxy_session_id).await { + error!( + "HLS session lifecycle cleanup failed: session={} error={err}", + safe_proxy_session_id(proxy_session_id) + ); + } else { + debug!("HLS session lifecycle expired: session={}", safe_proxy_session_id(proxy_session_id)); + debug!("HLS lifecycle state snapshot: trigger=session-expired {}", self.debug_state_summary().await); + } + } + } + + pub async fn debug_state_summary(&self) -> String { + let sessions = self.sessions.list_sessions().await; + let access_leases = self.access_leases.read().await.len(); + let qos_access_leases = self.qos.len().await; + let repair = self.segment_repair.stats().await; + let mut segments = 0_usize; + let mut maps = 0_usize; + let mut transient_resources = 0_usize; + let mut transient_objects = 0_usize; + let mut active_origin_work = 0_usize; + let mut active_segment_fetches = 0_usize; + let mut active_map_fetches = 0_usize; + for session in &sessions { + let session = session.read().await; + segments = segments.saturating_add(session.segments.len()); + maps = maps.saturating_add(session.maps.len()); + transient_resources = transient_resources.saturating_add(session.transient.resources.len()); + transient_objects = transient_objects.saturating_add(session.transient.object_cache.len()); + active_origin_work = active_origin_work.saturating_add(session.activity.active_origin_work_count); + active_segment_fetches = active_segment_fetches.saturating_add(session.active_segment_fetches); + active_map_fetches = active_map_fetches.saturating_add(session.active_map_fetches); + } + format!( + "sessions={} access_leases={} qos_access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} segments={} maps={} transient_resources={} transient_objects={} active_origin_work={} active_segment_fetches={} active_map_fetches={}", + sessions.len(), + access_leases, + qos_access_leases, + repair.windows, + repair.generations, + repair.checked_candidates, + repair.metadata, + repair.object_metadata, + repair.locks, + repair.watchdog_metadata, + repair.watchdog_locks, + segments, + maps, + transient_resources, + transient_objects, + active_origin_work, + active_segment_fetches, + active_map_fetches + ) + } + + pub async fn run_garbage_collection_once(&self, now_ms: u64) -> io::Result { + if !self.is_enabled() { + return Ok(super::GarbageCollectionReport::default()); + } + let report = self.gc.run_once(now_ms).await?; + self.cleanup_after_garbage_collection(&report).await; + Ok(report) + } + + pub async fn sync_session_access_lease_count_and_detach_if_needed( + &self, + active_users: &Arc, + _active_provider: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) { + let snapshot = self.access_lease_session_snapshot(proxy_session_id, now_ms).await; + for release in &snapshot.idle_releases { + active_users + .release_session_streams_and_counted_reservation(&release.username, &release.user_session_token) + .await; + debug!( + "HLS access lease idled: lease={} proxy_session={} session={}", + super::safe_hls_access_lease_id(&release.lease_id), + safe_proxy_session_id(proxy_session_id), + super::safe_user_session_token(&release.user_session_token) + ); + } + { + let mut session = session.write().await; + session.activity.active_access_lease_count = snapshot.active_count; + session.reconcile_effective_origin_acquire_policy(snapshot.effective_origin_policy, now_ms); + } + } + + pub async fn sync_all_session_access_leases_and_detach_if_needed( + &self, + active_users: &Arc, + active_provider: &Arc, + now_ms: u64, + ) { + if !self.is_enabled() { + return; + } + for session in self.sessions.list_sessions().await { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + self.sync_session_access_lease_count_and_detach_if_needed( + active_users, + active_provider, + &session, + &proxy_session_id, + now_ms, + ) + .await; + } + } + + pub async fn deny_access_lease(&self, lease_id: &HlsAccessLeaseId) { + self.access_leases.write().await.deny_access_lease(lease_id); + } + + pub async fn get_or_create_session( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> HlsSessionHandle { + self.get_or_create_session_with_outcome(key, reverse_proxy_rewrite_secret, now_ms).await.0 + } + + pub async fn get_or_create_session_with_outcome( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let origin_source = HlsOriginSource::from_session_key(&key); + self.get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) + .await + } + + pub async fn get_or_create_session_with_source_and_outcome( + &self, + key: HlsSessionKey, + origin_source: HlsOriginSource, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let (session, outcome) = self + .sessions + .get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) + .await; + let (proxy_session_id, session_key) = { + let session_guard = session.read().await; + (safe_proxy_session_id(&session_guard.proxy_session_id), safe_session_key(&session_guard.key)) + }; + match outcome { + HlsSessionStoreOutcome::Created => { + self.metrics.record_session_created(); + info!("HLS session created: session={session_key} proxy_session_id={proxy_session_id}"); + } + HlsSessionStoreOutcome::Reused => { + self.metrics.record_session_reused(); + debug!("HLS session reused: session={session_key} proxy_session_id={proxy_session_id}"); + } + } + self.schedule_session_idle_for_handle(&session).await; + session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); + (session, outcome) + } +} + +impl Default for HlsProxyManager { + fn default() -> Self { Self::new() } +} + +pub fn exec_hls_lifecycle(app_state: &Arc, cancel_token: &CancellationToken) { + let hls_proxy = Arc::clone(&app_state.hls_proxy); + let active_users = Arc::clone(&app_state.active_users); + let active_provider = Arc::clone(&app_state.active_provider); + let cancel_token = cancel_token.clone(); + tokio::spawn(async move { + while let Some(event) = hls_proxy.lifecycle().next_event(&cancel_token).await { + let now_ms = current_time_millis(); + hls_proxy.handle_lifecycle_event(&active_users, &active_provider, event, now_ms).await; + } + }); +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::HlsProxyManager; + use crate::{ + api::model::HlsSessionKey, + model::{AppConfig, Config, HlsCacheConfig, MediaToolCapabilities, ReverseProxyConfig, SourcesConfig}, + utils::FileLockManager, + }; + use arc_swap::{ArcSwap, ArcSwapOption}; + use shared::model::{ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode, ReverseProxyConfigDto}; + use std::sync::Arc; + + fn empty_paths() -> ConfigPaths { + ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + } + } + + fn test_app_config(config: Config) -> AppConfig { + AppConfig { + config: Arc::new(ArcSwap::from_pointee(config)), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(empty_paths())), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [7; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } + } + + fn config_with_hls_cache(hls_cache: HlsCacheConfigDto) -> Config { + Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(hls_cache), + ..ReverseProxyConfigDto::default() + })), + ..Config::default() + } + } + + #[tokio::test] + async fn update_config_applies_hls_runtime_settings_to_existing_manager() { + let initial_dto = HlsCacheConfigDto { + cache_path: Some("/tmp/tuliprox/hls-a".to_string()), + max_segments_prefetch: 1, + ..Default::default() + }; + let initial_config = HlsCacheConfig::from(&initial_dto); + let manager = HlsProxyManager::with_hls_cache_config(&initial_config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; + + let mut updated_dto = initial_dto.clone(); + updated_dto.max_segments_prefetch = 4; + updated_dto.max_concurrent_segment_fetches_per_session = 5; + updated_dto.max_concurrent_segment_fetches_global = 6; + updated_dto.origin_manifest_timeout_ms = 1_234; + updated_dto.origin_segment_timeout_ms = 5_678; + updated_dto.cache_duration = 99; + updated_dto.session_idle_timeout = 55; + updated_dto.manifest_recovery_burst.level = shared::model::HlsManifestRecoveryBurstLevel::Balanced; + updated_dto.strip = HlsStripConfigDto { mode: HlsStripMode::Seconds, value: 7 }; + let app_config = test_app_config(config_with_hls_cache(updated_dto)); + + manager.update_config(&app_config).await; + + assert_eq!(manager.segment_fetch_policy().max_prefetch_queue_depth, 4); + assert_eq!(manager.segment_fetch_policy().max_session_segment_fetches, 5); + assert_eq!(manager.segment_fetch_policy().max_global_segment_fetches, 6); + assert_eq!(manager.segment_fetch_policy().origin_segment_timeout_ms, 5_678); + assert_eq!(manager.origin_manifest_timeout_ms(), 1_234); + assert_eq!(manager.cache_duration_seconds(), 99); + assert_eq!(manager.transient_resource_ttl_ms(), 99_000); + assert_eq!(manager.session_idle_timeout_ms(), 55_000); + assert_eq!( + manager.manifest_recovery_burst().level, + HlsManifestRecoveryBurstLevel::Balanced + ); + assert_eq!(manager.strip().mode, HlsStripMode::Seconds); + assert_eq!(manager.strip().value, 7); + assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); + } + + #[tokio::test] + async fn optional_hls_config_controls_runtime_enabled_state() { + let manager = HlsProxyManager::from_hls_cache_config(None); + + assert!(!manager.is_enabled()); + assert_eq!( + manager + .run_garbage_collection_once(1_000) + .await + .expect("disabled gc should no-op"), + super::super::GarbageCollectionReport::default() + ); + + let app_config = test_app_config(config_with_hls_cache(HlsCacheConfigDto::default())); + manager.update_config(&app_config).await; + + assert!(manager.is_enabled()); + } + + #[tokio::test] + async fn update_config_cache_path_change_clears_runtime_cache_state() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let old_cache = temp_dir.path().join("old"); + let new_cache = temp_dir.path().join("new"); + let initial_dto = + HlsCacheConfigDto { cache_path: Some(old_cache.to_string_lossy().to_string()), ..Default::default() }; + let initial_config = HlsCacheConfig::from(&initial_dto); + let manager = HlsProxyManager::with_hls_cache_config(&initial_config); + let _ = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; + assert_eq!(manager.sessions().len().await, 1); + + let mut updated_dto = initial_dto; + updated_dto.cache_path = Some(new_cache.to_string_lossy().to_string()); + let app_config = test_app_config(config_with_hls_cache(updated_dto)); + + manager.update_config(&app_config).await; + + assert!(manager.sessions().is_empty().await); + assert_eq!(manager.segment_cache().cache_path(), new_cache); + } +} diff --git a/backend/src/api/model/hls_cache/manifest_commit.rs b/backend/src/api/model/hls_cache/manifest_commit.rs new file mode 100644 index 000000000..23026b2aa --- /dev/null +++ b/backend/src/api/model/hls_cache/manifest_commit.rs @@ -0,0 +1,193 @@ +use super::{ + HlsAccountBindingProtection, HlsFreshManifestRequiredReason, HlsManifestCommitRequirement, + HlsOriginAccountBinding, HlsSession, HlsSessionHandle, HlsSessionMode, HlsSessionStoreOutcome, +}; +use std::time::Duration; + +#[derive(Clone)] +pub enum HlsCommittedManifestBody { + Normal(String), + Transient(String), +} + +struct HlsCommittedManifestCandidate { + body: HlsCommittedManifestBody, + rendered_at_ms: u64, + valid_until_ms: Option, +} + +#[cfg_attr(not(test), allow(dead_code))] +#[derive(Clone, Copy)] +pub enum HlsCachedManifestPolicy { + CommittedOnly, + AllowInitialNoMediaYet, +} + +#[derive(Clone, Copy)] +pub struct HlsCachedManifestOptions { + pub wait_timeout: Duration, + policy: HlsCachedManifestPolicy, + newer_than_rendered_at_ms: Option, +} + +impl HlsCachedManifestOptions { + #[cfg_attr(not(test), allow(dead_code))] + pub const fn committed_only(wait_timeout: Duration) -> Self { + Self { wait_timeout, policy: HlsCachedManifestPolicy::CommittedOnly, newer_than_rendered_at_ms: None } + } + + pub const fn initial(wait_timeout: Duration) -> Self { + Self { wait_timeout, policy: HlsCachedManifestPolicy::AllowInitialNoMediaYet, newer_than_rendered_at_ms: None } + } + + pub const fn requiring_newer_manifest(mut self, rendered_at_ms: u64) -> Self { + self.newer_than_rendered_at_ms = Some(rendered_at_ms); + self + } + + const fn requires_newer_manifest(self) -> bool { self.newer_than_rendered_at_ms.is_some() } +} + +pub const fn hls_cached_manifest_options_for_requirement( + wait_timeout: Duration, + requirement: HlsManifestCommitRequirement, + previous_rendered_at_ms: u64, +) -> HlsCachedManifestOptions { + let options = HlsCachedManifestOptions::initial(wait_timeout); + match requirement { + HlsManifestCommitRequirement::CommittedManifestAllowed => options, + HlsManifestCommitRequirement::FreshCommitRequired { .. } => { + options.requiring_newer_manifest(previous_rendered_at_ms) + } + } +} + +pub fn hls_committed_manifest_body_for_request( + session: &HlsSession, + options: HlsCachedManifestOptions, + started_at_ms: u64, + now_ms: u64, +) -> Option { + let protection = session.account_binding_protection(now_ms); + hls_committed_manifest_candidate(session).and_then(|candidate| { + (can_serve_committed_manifest( + session, + &candidate, + protection, + options.policy, + candidate.rendered_at_ms >= started_at_ms, + now_ms, + ) && manifest_rendered_after_required_boundary(Some(candidate.rendered_at_ms), options)) + .then_some(candidate.body) + }) +} + +pub fn hls_should_wait_for_initial_manifest_commit( + session: &HlsSession, + selected_committed_body: bool, + should_wait: bool, + options: HlsCachedManifestOptions, + now_ms: u64, +) -> bool { + let protection = session.account_binding_protection(now_ms); + !selected_committed_body + && (matches!(protection, HlsAccountBindingProtection::NoMediaYet | HlsAccountBindingProtection::Expired) + || options.requires_newer_manifest()) + && should_wait + && !options.wait_timeout.is_zero() +} + +pub async fn hls_manifest_commit_requirement( + session: &HlsSessionHandle, + session_outcome: HlsSessionStoreOutcome, + handoff_previous_rendered_at_ms: Option, + now_ms: u64, +) -> HlsManifestCommitRequirement { + if handoff_previous_rendered_at_ms.is_some() { + return HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ProvisioningHandoff, + }; + } + if matches!(session_outcome, HlsSessionStoreOutcome::Created) { + return HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ColdStart, + }; + } + + let session = session.read().await; + if let Some(reason) = session.fresh_manifest_commit_required { + return HlsManifestCommitRequirement::FreshCommitRequired { reason }; + } + if hls_committed_manifest_available_for_request(&session, now_ms) { + HlsManifestCommitRequirement::CommittedManifestAllowed + } else { + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + } + } +} + +fn hls_committed_manifest_available_for_request(session: &HlsSession, now_ms: u64) -> bool { + let protection = session.account_binding_protection(now_ms); + hls_committed_manifest_candidate(session).is_some_and(|candidate| { + can_serve_committed_manifest( + session, + &candidate, + protection, + HlsCachedManifestPolicy::AllowInitialNoMediaYet, + false, + now_ms, + ) + }) +} + +fn hls_committed_manifest_candidate(session: &HlsSession) -> Option { + match session.mode { + HlsSessionMode::NormalCacheTimeline => { + session.last_rendered_manifest.as_ref().map(|rendered| HlsCommittedManifestCandidate { + body: HlsCommittedManifestBody::Normal(rendered.body.clone()), + rendered_at_ms: rendered.rendered_at_ms, + valid_until_ms: Some(rendered.valid_until_ms), + }) + } + HlsSessionMode::TransientPassthrough { .. } => { + session.transient.last_manifest_body.as_ref().map(|body| HlsCommittedManifestCandidate { + body: HlsCommittedManifestBody::Transient(body.clone()), + rendered_at_ms: session.transient.last_manifest_rendered_at_ms.unwrap_or_default(), + valid_until_ms: session.transient.last_manifest_valid_until_ms, + }) + } + } +} + +fn manifest_rendered_after_required_boundary(rendered_at_ms: Option, options: HlsCachedManifestOptions) -> bool { + let Some(boundary) = options.newer_than_rendered_at_ms else { + return true; + }; + rendered_at_ms.is_some_and(|rendered_at_ms| rendered_at_ms > boundary) +} + +fn can_serve_committed_manifest( + session: &HlsSession, + candidate: &HlsCommittedManifestCandidate, + protection: HlsAccountBindingProtection, + policy: HlsCachedManifestPolicy, + refreshed_after_wait_started: bool, + now_ms: u64, +) -> bool { + match protection { + HlsAccountBindingProtection::HardActive { .. } | HlsAccountBindingProtection::SoftActive { .. } => true, + HlsAccountBindingProtection::NoMediaYet => { + matches!(policy, HlsCachedManifestPolicy::AllowInitialNoMediaYet) + && committed_manifest_valid_at(candidate, now_ms) + } + HlsAccountBindingProtection::Expired => matches!(policy, HlsCachedManifestPolicy::AllowInitialNoMediaYet) + && (refreshed_after_wait_started + || (session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_active) + && committed_manifest_valid_at(candidate, now_ms))), + } +} + +fn committed_manifest_valid_at(candidate: &HlsCommittedManifestCandidate, now_ms: u64) -> bool { + candidate.valid_until_ms.is_some_and(|valid_until_ms| now_ms <= valid_until_ms) +} diff --git a/backend/src/api/model/hls_cache/manifest_fetch.rs b/backend/src/api/model/hls_cache/manifest_fetch.rs new file mode 100644 index 000000000..7677fa5c1 --- /dev/null +++ b/backend/src/api/model/hls_cache/manifest_fetch.rs @@ -0,0 +1,1858 @@ +use super::{ + extract_hls_provider_session_header_map, safe_origin_log_value, safe_session_key, HlsAccountBindingProtection, + HlsBoundAccountAcquireErrorKind, HlsSessionHandle, HlsSessionMode, TimelineMapError, +}; +use crate::{ + model::{ + resolve_provider_scheme_url_with_provider_index, AppConfig, ConfigProvider, HlsManifestRecoveryBurstConfig, + InputSource, StripConfig, + }, + processing::parser::hls::origin_manifest::{ + parse_manifest_timing, parse_origin_manifest_timeline, parse_origin_media_manifest, OriginManifestParseOutcome, + ParsedOriginManifestTimeline, + }, + utils::request::{ + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_input_with_retry_and_provider_policy_with_options_result, RequestFetchOptions, + }, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use futures::StreamExt; +use log::{debug, warn}; +use reqwest::Client; +use shared::{ + model::{HlsManifestRecoveryBurstLevel, HlsManifestRecoveryBurstPlan, InputFetchMethod, HlsStripMode}, + utils::sanitize_sensitive_info, +}; +use std::{collections::HashMap, fmt, future::Future, sync::Arc, time::Duration}; +use tokio::{task::JoinSet, time::timeout}; +use url::Url; + +const MAX_MANUAL_REDIRECTS: usize = 10; +const DEFAULT_HLS_TARGET_DURATION_SECS: u32 = 15; +const HLS_MANIFEST_HOST_SWITCH_BASE_WINDOW_SEGMENTS: u32 = 3; +const HLS_MANIFEST_HOST_SWITCH_MAX_FAILURE_THRESHOLD: u32 = 5; +const DEFAULT_HLS_SESSION_IDLE_TIMEOUT_SECS: u64 = 300; +const MAX_HLS_MANIFEST_BYTES: usize = 2 * 1024 * 1024; + +/// Origin manifest entrypoint snapshot for live HLS refreshes. +#[derive(Clone)] +pub struct LiveHlsOriginEntry { + url: Url, + url_failover_provider: Option>, +} + +impl LiveHlsOriginEntry { + pub fn parse(url: &str) -> Option { Self::parse_with_url_failover_provider(url, None) } + + pub fn parse_with_url_failover_provider( + url: &str, + url_failover_provider: Option>, + ) -> Option { + Url::parse(url).ok().map(|url| Self { url, url_failover_provider }) + } + + pub fn url(&self) -> &Url { &self.url } + + pub fn url_failover_provider(&self) -> Option<&Arc> { self.url_failover_provider.as_ref() } + + pub fn to_input_source(&self) -> InputSource { + InputSource { + name: Arc::::from("hls-origin"), + url: self.url.to_string(), + // In this HLS context, InputSource.provider is the URL-failover provider from source.yml, + // not a runtime origin-account provider. + provider: self.url_failover_provider.clone(), + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::new(), + } + } +} + +impl fmt::Debug for LiveHlsOriginEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LiveHlsOriginEntry") + .field("scheme", &self.url.scheme()) + .field("host", &self.url.host_str().unwrap_or("")) + .field("path", &"") + .field("url_failover_provider", &self.url_failover_provider.as_ref().map(|provider| provider.name.as_ref())) + .finish() + } +} + +/// Fixed retry policy for HLS origin manifest refreshes. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct RetryPolicy { + pub delays_ms: [u64; 5], + pub jitter_max_ms: u64, +} + +impl Default for RetryPolicy { + fn default() -> Self { Self { delays_ms: [0, 100, 250, 500, 750], jitter_max_ms: 100 } } +} + +impl RetryPolicy { + pub fn delay_for_attempt_ms(&self, attempt_index: usize, jitter_ms: u64) -> Option { + self.delays_ms.get(attempt_index).map(|base| base.saturating_add(jitter_ms.min(self.jitter_max_ms))) + } + + pub(crate) fn attempt_count(&self) -> usize { self.delays_ms.len() } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum OriginManifestStatusClass { + Success, + Retryable, + PermanentFailure, + NonRetryableFailure, +} + +pub fn classify_origin_manifest_status(status: StatusCode) -> OriginManifestStatusClass { + if status.is_success() { + return OriginManifestStatusClass::Success; + } + if status.is_server_error() + || matches!( + status, + StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_EARLY + | StatusCode::TOO_MANY_REQUESTS + ) + { + return OriginManifestStatusClass::Retryable; + } + if matches!( + status, + StatusCode::BAD_REQUEST + | StatusCode::UNAUTHORIZED + | StatusCode::FORBIDDEN + | StatusCode::NOT_FOUND + | StatusCode::GONE + ) { + return OriginManifestStatusClass::PermanentFailure; + } + OriginManifestStatusClass::NonRetryableFailure +} + +#[derive(Debug)] +pub enum OriginManifestFetchError { + PermanentStatus(StatusCode), + RetryableStatus(StatusCode, Option), + RetryExhausted, + NonRetryableStatus(StatusCode), + Request(String), + Redirect(String), + Timeout, + ProviderUnavailable(HlsBoundAccountAcquireErrorKind), +} + +#[derive(Debug)] +pub(crate) enum HlsManifestCommitError { + TimelineRejected { reason: HlsManifestRejectLogReason }, + RetryCurrentTarget, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum HlsManifestAcceptanceRejectReason { + MissingPinnedTarget, + HostSwitchPending { failures: u32, threshold: u32 }, + MissingOriginHighwater, + ForwardTooFar { previous: u64, origin: u64, window: Option }, + BackwardOutsideRollover { previous: u64, origin: u64, window: Option }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum HlsManifestRejectLogReason { + MissingPinnedTarget, + HostSwitchPending { failures: u32, threshold: u32 }, + MissingOriginHighwater, + ForwardTooFar { previous: u64, origin: u64, window: Option }, + BackwardOutsideRollover { previous: u64, origin: u64, window: Option }, + PinnedHostRecoveryRejected, + UnsupportedSegmentExtension, + UnsupportedMapExtension, + ProxySequenceOverflow, + ProxyMapIdOverflow, + MalformedTransientTimeline, +} + +impl HlsManifestRejectLogReason { + pub(crate) fn status_label(&self) -> String { + match self { + Self::MissingPinnedTarget => "missing-pinned-target".to_string(), + Self::HostSwitchPending { failures, threshold } => { + format!("host-switch-pending failures={failures} threshold={threshold}") + } + Self::MissingOriginHighwater => "missing-origin-highwater".to_string(), + Self::ForwardTooFar { previous, origin, window } => { + format!( + "forward-too-far previous={previous} origin={origin} window={}", + format_optional_highwater(*window) + ) + } + Self::BackwardOutsideRollover { previous, origin, window } => { + format!( + "backward-outside-rollover previous={previous} origin={origin} window={}", + format_optional_highwater(*window) + ) + } + Self::PinnedHostRecoveryRejected => "pinned-host-recovery-rejected".to_string(), + Self::UnsupportedSegmentExtension => "unsupported-segment-extension".to_string(), + Self::UnsupportedMapExtension => "unsupported-map-extension".to_string(), + Self::ProxySequenceOverflow => "proxy-sequence-overflow".to_string(), + Self::ProxyMapIdOverflow => "proxy-map-id-overflow".to_string(), + Self::MalformedTransientTimeline => "malformed-transient-timeline".to_string(), + } + } +} + +impl From for HlsManifestRejectLogReason { + fn from(reason: HlsManifestAcceptanceRejectReason) -> Self { + match reason { + HlsManifestAcceptanceRejectReason::MissingPinnedTarget => Self::MissingPinnedTarget, + HlsManifestAcceptanceRejectReason::HostSwitchPending { failures, threshold } => { + Self::HostSwitchPending { failures, threshold } + } + HlsManifestAcceptanceRejectReason::MissingOriginHighwater => Self::MissingOriginHighwater, + HlsManifestAcceptanceRejectReason::ForwardTooFar { previous, origin, window } => { + Self::ForwardTooFar { previous, origin, window } + } + HlsManifestAcceptanceRejectReason::BackwardOutsideRollover { previous, origin, window } => { + Self::BackwardOutsideRollover { previous, origin, window } + } + } + } +} + +impl From for HlsManifestRejectLogReason { + fn from(err: TimelineMapError) -> Self { + match err { + TimelineMapError::UnsupportedSegmentExtension => Self::UnsupportedSegmentExtension, + TimelineMapError::UnsupportedMapExtension => Self::UnsupportedMapExtension, + TimelineMapError::ProxySequenceOverflow => Self::ProxySequenceOverflow, + TimelineMapError::ProxyMapIdOverflow => Self::ProxyMapIdOverflow, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsManifestCommitAcceptanceMode { + StrictPinnedHost, + AllowHeldHostSwitchCandidate, + FreshBaseline, +} + +#[derive(Clone, Eq, PartialEq)] +pub struct FetchedOriginManifest { + pub body: String, + pub final_manifest_url: String, + pub resolved_request_url: String, + pub redirect_host: Option, + pub provider_url_index: Option, + pub provider_session_headers: HeaderMap, + pub status: StatusCode, + pub attempts: usize, +} + +impl fmt::Debug for FetchedOriginManifest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FetchedOriginManifest") + .field("body_len", &self.body.len()) + .field("final_manifest_url", &"") + .field("resolved_request_url", &"") + .field("redirect_host", &self.redirect_host) + .field("provider_url_index", &self.provider_url_index) + .field("provider_session_headers_len", &self.provider_session_headers.len()) + .field("status", &self.status) + .field("attempts", &self.attempts) + .finish() + } +} + +impl FetchedOriginManifest { + pub(crate) fn with_attempts(mut self, attempts: usize) -> Self { + self.attempts = attempts; + self + } +} + +#[derive(Clone)] +pub(crate) struct HlsOriginManifestFetchContext { + pub app_config: Arc, + pub session: HlsSessionHandle, + pub origin_entry: LiveHlsOriginEntry, + pub headers: HeaderMap, + pub client: Client, + pub no_redirect_client: Client, + pub use_manual_redirects: bool, + pub origin_manifest_timeout_ms: u64, + pub manifest_recovery_burst: HlsManifestRecoveryBurstConfig, + pub retry_policy: RetryPolicy, +} + +enum HlsOriginManifestFetchMode<'a> { + InitialGlobalPolicy, + RecoveryDirectTarget { + target_url: &'a Url, + provider_url_index: Option, + reason: Option<&'a HlsManifestRejectLogReason>, + log_context: ManifestRecoveryAttemptLogContext, + }, +} + +pub(crate) struct HlsOriginManifestFetchRequest<'a> { + context: &'a HlsOriginManifestFetchContext, + mode: HlsOriginManifestFetchMode<'a>, +} + +impl<'a> HlsOriginManifestFetchRequest<'a> { + pub(crate) const fn initial_global_policy(context: &'a HlsOriginManifestFetchContext) -> Self { + Self { context, mode: HlsOriginManifestFetchMode::InitialGlobalPolicy } + } + + const fn recovery_direct_target( + context: &'a HlsOriginManifestFetchContext, + target_url: &'a Url, + provider_url_index: Option, + reason: Option<&'a HlsManifestRejectLogReason>, + log_context: ManifestRecoveryAttemptLogContext, + ) -> Self { + Self { + context, + mode: HlsOriginManifestFetchMode::RecoveryDirectTarget { + target_url, + provider_url_index, + reason, + log_context, + }, + } + } +} + +enum HlsManifestRecoveryAttemptError { + Fetch(OriginManifestFetchError), + Rejected(HlsManifestRejectLogReason), + Committed(T), +} + +#[derive(Debug)] +struct HlsManifestRecoveryCandidate { + candidate_index: usize, + fetched: FetchedOriginManifest, + report: HlsManifestRecoveryCandidateScoreReport, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HlsManifestRecoveryCandidateScoreReport { + pub(crate) media_sequence: u64, + pub(crate) quality: HlsManifestOriginQuality, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum HlsManifestOriginQualityScore { + Rejected, + OtherHostUnchanged, + SameHostUnchanged, + OtherHostRolloverCandidate, + SameHostRolloverCandidate, + OtherHostRebaseCandidate, + OtherHostPlausibleForward, + OtherHostNextSequence, + SameHostRebase, + SameHostPlausibleForward, + SameHostNextSequence, +} + +impl HlsManifestOriginQualityScore { + pub(crate) const fn rank(self) -> u16 { + match self { + Self::Rejected => 0, + Self::OtherHostUnchanged => 10, + Self::SameHostUnchanged => 20, + Self::OtherHostRolloverCandidate => 35, + Self::SameHostRolloverCandidate => 50, + Self::OtherHostRebaseCandidate => 60, + Self::OtherHostPlausibleForward => 65, + Self::OtherHostNextSequence => 75, + Self::SameHostRebase => 85, + Self::SameHostPlausibleForward => 90, + Self::SameHostNextSequence => 100, + } + } + + const fn as_log_value(self) -> &'static str { + match self { + Self::Rejected => "rejected", + Self::OtherHostUnchanged => "other-host-unchanged", + Self::SameHostUnchanged => "same-host-unchanged", + Self::OtherHostRolloverCandidate => "other-host-rollover-candidate", + Self::SameHostRolloverCandidate => "same-host-rollover-candidate", + Self::OtherHostRebaseCandidate => "other-host-rebase-candidate", + Self::OtherHostPlausibleForward => "other-host-plausible-forward", + Self::OtherHostNextSequence => "other-host-next-sequence", + Self::SameHostRebase => "same-host-rebase", + Self::SameHostPlausibleForward => "same-host-plausible-forward", + Self::SameHostNextSequence => "same-host-next-sequence", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsManifestOriginRelation { + Initial, + SameRedirectHost, + OtherRedirectHost, + UnknownHost, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsManifestSequenceRelation { + NoPreviousHighwater, + NoOriginHighwater, + Rebase, + Same, + Next, + PlausibleForward, + ForwardTooFar, + RolloverCandidate, + Backward, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsManifestContinuityMode { + StrictContinuity, + RebaseAllowed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HlsManifestOriginQuality { + pub(crate) score: HlsManifestOriginQualityScore, + pub(crate) continuity_mode: HlsManifestContinuityMode, + pub(crate) host_relation: HlsManifestOriginRelation, + pub(crate) sequence_relation: HlsManifestSequenceRelation, + pub(crate) effective_host: Option, + pub(crate) origin_highwater: Option, + pub(crate) previous_highwater: Option, + pub(crate) allowed_forward_window: Option, + pub(crate) should_increment_stall_counter: bool, + pub(crate) should_reset_stall_counter: bool, + pub(crate) requires_handoff_discontinuity: bool, + pub(crate) reject_reason: Option, +} + +pub(crate) async fn fetch_hls_origin_manifest_request( + request: HlsOriginManifestFetchRequest<'_>, +) -> Result { + match request.mode { + HlsOriginManifestFetchMode::InitialGlobalPolicy => { + fetch_hls_origin_manifest_initial_global_policy(request.context).await + } + HlsOriginManifestFetchMode::RecoveryDirectTarget { target_url, provider_url_index, reason, log_context } => { + fetch_hls_origin_manifest_recovery_direct_target( + request.context, + target_url, + provider_url_index, + reason, + log_context, + ) + .await + } + } +} + +async fn fetch_hls_origin_manifest_initial_global_policy( + context: &HlsOriginManifestFetchContext, +) -> Result { + log_manifest_initial_attempt(context).await; + let input_source = context.origin_entry.to_input_source(); + let account_binding = { + let session = context.session.read().await; + if session.origin_account_binding.is_some() { + "present" + } else { + "absent" + } + }; + debug!( + "HLS origin manifest request started: account_binding={account_binding} origin_entry={}", + safe_origin_log_value(input_source.url.as_str()) + ); + let fetch_options = RequestFetchOptions::with_attempt_idle_timeout(Duration::from_millis( + context.origin_manifest_timeout_ms.max(1), + )); + let response_result = if context.use_manual_redirects { + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &context.app_config, + &context.no_redirect_client, + &input_source, + Some(&context.headers), + context.origin_entry.url(), + MAX_MANUAL_REDIRECTS, + fetch_options, + ) + .await + } else { + send_input_with_retry_and_provider_policy_with_options_result( + &context.app_config, + &context.client, + &input_source, + Some(&context.headers), + context.origin_entry.url(), + fetch_options, + ) + .await + }; + let response_result = response_result.map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + let provider_url_index = response_result.provider_url_index; + let resolved_request_url = + resolved_hls_manifest_request_url_from_input(&input_source, provider_url_index, context.origin_entry.url()); + response_to_fetched_manifest( + response_result.response, + provider_url_index, + resolved_request_url, + context.origin_manifest_timeout_ms, + ) + .await +} + +pub(crate) async fn score_hls_manifest_candidate_for_selection_log( + context: &HlsOriginManifestFetchContext, + fetched: &FetchedOriginManifest, + acceptance_mode: HlsManifestCommitAcceptanceMode, +) -> Option { + let session = context.session.read().await; + let timeline = parse_manifest_timeline_for_recovery_scoring(&session, fetched).ok()?; + Some(HlsManifestRecoveryCandidateScoreReport { + media_sequence: timeline.origin_manifest_sequence, + quality: evaluate_manifest_origin_quality_with_mode( + &session, + fetched, + timeline, + context, + current_time_millis(), + acceptance_mode, + ), + }) +} + +pub(crate) async fn retry_hls_origin_manifest_recovery_chain( + context: &HlsOriginManifestFetchContext, + target_url: Url, + provider_url_index: Option, + mut reject_reason: Option, + mut commit: C, +) -> Result +where + C: FnMut(FetchedOriginManifest, HlsManifestCommitAcceptanceMode) -> Fut, + Fut: Future>, +{ + let attempts = context.retry_policy.attempt_count(); + let mut last_error = OriginManifestFetchError::RetryExhausted; + + for attempt_index in 0..attempts { + let delay_ms = { + let jitter = if context.retry_policy.jitter_max_ms == 0 { + 0 + } else { + fastrand::u64(0..=context.retry_policy.jitter_max_ms) + }; + context.retry_policy.delay_for_attempt_ms(attempt_index, jitter).unwrap_or_default() + }; + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + let attempt_plan = HlsManifestRecoveryAttemptPlan { + target_url: &target_url, + provider_url_index, + attempt_index, + attempts, + reject_reason: reject_reason.as_ref(), + acceptance_mode: HlsManifestCommitAcceptanceMode::StrictPinnedHost, + }; + match fetch_and_commit_manifest_recovery_attempt(context, attempt_plan, &mut commit).await { + HlsManifestRecoveryAttemptError::Committed(committed) => return Ok(committed), + HlsManifestRecoveryAttemptError::Rejected(reason) if attempt_index + 1 < attempts => { + log_manifest_retry_scheduled( + context, + attempt_index, + attempts, + next_retry_delay_ms(&context.retry_policy, attempt_index, None), + Some(&reason), + None, + ) + .await; + reject_reason = Some(reason); + last_error = OriginManifestFetchError::RetryExhausted; + } + HlsManifestRecoveryAttemptError::Rejected(_reason) => { + return Err(OriginManifestFetchError::RetryExhausted); + } + HlsManifestRecoveryAttemptError::Fetch(err) + if is_hls_retryable_manifest_reject_fetch_error(&err) && attempt_index + 1 < attempts => + { + log_manifest_retry_scheduled( + context, + attempt_index, + attempts, + next_retry_delay_ms(&context.retry_policy, attempt_index, None), + None, + Some(&err), + ) + .await; + last_error = err; + } + HlsManifestRecoveryAttemptError::Fetch(err) => return Err(err), + } + } + + Err(last_error) +} + +struct HlsManifestRecoveryAttemptPlan<'a> { + target_url: &'a Url, + provider_url_index: Option, + attempt_index: usize, + attempts: usize, + reject_reason: Option<&'a HlsManifestRejectLogReason>, + acceptance_mode: HlsManifestCommitAcceptanceMode, +} + +async fn fetch_and_commit_manifest_recovery_attempt( + context: &HlsOriginManifestFetchContext, + plan: HlsManifestRecoveryAttemptPlan<'_>, + commit: &mut C, +) -> HlsManifestRecoveryAttemptError +where + C: FnMut(FetchedOriginManifest, HlsManifestCommitAcceptanceMode) -> Fut, + Fut: Future>, +{ + let burst_plan = recovery_burst_plan(context, plan.attempt_index); + let candidates = burst_plan.total_candidates(); + if candidates == 1 { + let fetched = match fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::recovery_direct_target( + context, + plan.target_url, + plan.provider_url_index, + plan.reject_reason, + ManifestRecoveryAttemptLogContext::single(plan.attempt_index, plan.attempts), + )) + .await + { + Ok(fetched) => fetched, + Err(err) => return HlsManifestRecoveryAttemptError::Fetch(err), + }; + let report = score_manifest_recovery_candidate_with_logging(context, 0, candidates, &fetched).await.ok(); + let committed = commit(fetched.with_attempts(plan.attempt_index + 1), plan.acceptance_mode).await; + match (committed, report.as_ref()) { + (Ok(committed), Some(report)) => { + log_manifest_recovery_selected(context, 0, candidates, report).await; + HlsManifestRecoveryAttemptError::Committed(committed) + } + (Ok(committed), None) => HlsManifestRecoveryAttemptError::Committed(committed), + (Err(err), _) => HlsManifestRecoveryAttemptError::Rejected(commit_error_to_retry_reason(&err)), + } + } else { + fetch_and_commit_manifest_recovery_burst_attempt(context, plan, burst_plan, commit).await + } +} + +#[allow(clippy::too_many_arguments)] +async fn fetch_and_commit_manifest_recovery_burst_attempt( + context: &HlsOriginManifestFetchContext, + plan: HlsManifestRecoveryAttemptPlan<'_>, + burst_plan: HlsManifestRecoveryBurstPlan, + commit: &mut C, +) -> HlsManifestRecoveryAttemptError +where + C: FnMut(FetchedOriginManifest, HlsManifestCommitAcceptanceMode) -> Fut, + Fut: Future>, +{ + let (mut fetched_candidates, last_fetch_error, mut last_reject_reason) = fetch_manifest_recovery_burst_candidates( + context, + plan.target_url, + plan.provider_url_index, + plan.attempt_index, + plan.attempts, + plan.reject_reason, + burst_plan, + ) + .await; + let candidates = burst_plan.total_candidates(); + + fetched_candidates.sort_by(|left, right| { + right + .report + .quality + .score + .rank() + .cmp(&left.report.quality.score.rank()) + .then_with(|| right.report.quality.origin_highwater.cmp(&left.report.quality.origin_highwater)) + .then_with(|| left.candidate_index.cmp(&right.candidate_index)) + }); + for candidate in fetched_candidates { + let HlsManifestRecoveryCandidate { candidate_index, fetched, report } = candidate; + match commit(fetched.with_attempts(plan.attempt_index + 1), plan.acceptance_mode).await { + Ok(committed) => { + log_manifest_recovery_selected(context, candidate_index, candidates, &report).await; + return HlsManifestRecoveryAttemptError::Committed(committed); + } + Err(err) => { + let reason = commit_error_to_retry_reason(&err); + log_manifest_recovery_candidate_rejected( + context, + candidate_index, + candidates, + report.quality.effective_host.as_deref(), + report.quality.origin_highwater, + &reason, + ) + .await; + last_reject_reason = Some(reason); + } + } + } + + if let Some(reason) = last_reject_reason { + return HlsManifestRecoveryAttemptError::Rejected(reason); + } + HlsManifestRecoveryAttemptError::Fetch(last_fetch_error.unwrap_or(OriginManifestFetchError::RetryExhausted)) +} + +async fn fetch_manifest_recovery_burst_candidates( + context: &HlsOriginManifestFetchContext, + target_url: &Url, + provider_url_index: Option, + attempt_index: usize, + attempts: usize, + reject_reason: Option<&HlsManifestRejectLogReason>, + burst_plan: HlsManifestRecoveryBurstPlan, +) -> (Vec, Option, Option) { + let mut tasks = JoinSet::new(); + let candidates = burst_plan.total_candidates(); + for candidate_index in 0..candidates { + let context = context.clone(); + let target_url = target_url.clone(); + let reject_reason = reject_reason.cloned(); + tasks.spawn(async move { + let stagger_ms = + u64::try_from(burst_plan.slot_for_candidate(candidate_index)).unwrap_or_default().saturating_mul(100); + if stagger_ms > 0 { + tokio::time::sleep(Duration::from_millis(stagger_ms)).await; + } + let request = HlsOriginManifestFetchRequest::recovery_direct_target( + &context, + &target_url, + provider_url_index, + reject_reason.as_ref(), + ManifestRecoveryAttemptLogContext { attempt_index, attempts, candidate_index, candidates }, + ); + let result = fetch_hls_origin_manifest_request(request).await; + (candidate_index, result) + }); + } + + let mut last_fetch_error = None; + let mut last_reject_reason = None; + let mut fetched_candidates = Vec::new(); + while let Some(join_result) = tasks.join_next().await { + let Ok((candidate_index, result)) = join_result else { + last_fetch_error = Some(OriginManifestFetchError::Request("manifest recovery task failed".to_string())); + continue; + }; + match result { + Ok(fetched) => { + match score_manifest_recovery_candidate_with_logging(context, candidate_index, candidates, &fetched) + .await + { + Ok(report) => { + fetched_candidates.push(HlsManifestRecoveryCandidate { candidate_index, fetched, report }); + } + Err(reason) => last_reject_reason = Some(reason), + } + } + Err(err) => { + last_fetch_error = Some(err); + } + } + } + (fetched_candidates, last_fetch_error, last_reject_reason) +} + +async fn score_manifest_recovery_candidate_with_logging( + context: &HlsOriginManifestFetchContext, + candidate_index: usize, + candidates: usize, + fetched: &FetchedOriginManifest, +) -> Result { + let score_result = { + let session = context.session.read().await; + score_hls_manifest_recovery_candidate(&session, fetched, context) + }; + match score_result { + Ok(report) => { + log_manifest_recovery_candidate_scored(context, candidate_index, candidates, &report).await; + Ok(report) + } + Err(reason) => { + log_manifest_recovery_candidate_rejected( + context, + candidate_index, + candidates, + fetched_effective_manifest_host(fetched).as_deref(), + None, + &reason, + ) + .await; + Err(reason) + } + } +} + +pub(crate) fn score_hls_manifest_recovery_candidate( + session: &super::HlsSession, + fetched: &FetchedOriginManifest, + context: &HlsOriginManifestFetchContext, +) -> Result { + let timeline = parse_manifest_timeline_for_recovery_scoring(session, fetched)?; + let media_sequence = timeline.origin_manifest_sequence; + Ok(HlsManifestRecoveryCandidateScoreReport { + media_sequence, + quality: evaluate_manifest_origin_quality(session, fetched, timeline, context, current_time_millis()), + }) +} + +fn parse_manifest_timeline_for_recovery_scoring( + session: &super::HlsSession, + fetched: &FetchedOriginManifest, +) -> Result { + if matches!(session.mode, HlsSessionMode::TransientPassthrough { .. }) { + return parse_origin_manifest_timeline(&fetched.body) + .map_err(|_| HlsManifestRejectLogReason::MalformedTransientTimeline); + } + match parse_origin_media_manifest(&fetched.body, &fetched.final_manifest_url) { + OriginManifestParseOutcome::Normal(manifest) => Ok(ParsedOriginManifestTimeline { + origin_manifest_sequence: manifest.origin_manifest_sequence, + origin_manifest_segment_cnt: manifest.origin_manifest_segment_cnt, + }), + OriginManifestParseOutcome::TransientPassthrough { .. } => parse_origin_manifest_timeline(&fetched.body) + .map_err(|_| HlsManifestRejectLogReason::MalformedTransientTimeline), + } +} + +pub(crate) fn evaluate_manifest_origin_quality( + session: &super::HlsSession, + fetched: &FetchedOriginManifest, + timeline: ParsedOriginManifestTimeline, + context: &HlsOriginManifestFetchContext, + now_ms: u64, +) -> HlsManifestOriginQuality { + evaluate_manifest_origin_quality_with_mode( + session, + fetched, + timeline, + context, + now_ms, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + ) +} + +pub(crate) fn evaluate_manifest_origin_quality_with_mode( + session: &super::HlsSession, + fetched: &FetchedOriginManifest, + timeline: ParsedOriginManifestTimeline, + context: &HlsOriginManifestFetchContext, + now_ms: u64, + acceptance_mode: HlsManifestCommitAcceptanceMode, +) -> HlsManifestOriginQuality { + let effective_host = fetched_effective_manifest_host(fetched); + let fresh_baseline = matches!(acceptance_mode, HlsManifestCommitAcceptanceMode::FreshBaseline); + let host_relation = if fresh_baseline { + if effective_host.is_some() { + HlsManifestOriginRelation::Initial + } else { + HlsManifestOriginRelation::UnknownHost + } + } else { + match (session.last_effective_manifest_host.as_deref(), effective_host.as_deref()) { + (None, _) => HlsManifestOriginRelation::Initial, + (_, None) => HlsManifestOriginRelation::UnknownHost, + (Some(pinned), Some(effective)) if pinned == effective => HlsManifestOriginRelation::SameRedirectHost, + (Some(_), Some(_)) => HlsManifestOriginRelation::OtherRedirectHost, + } + }; + let origin_highwater = timeline.origin_highwater(); + let previous_highwater = if fresh_baseline { None } else { session.origin_seq_highwater }; + let continuity_mode = if fresh_baseline { + HlsManifestContinuityMode::RebaseAllowed + } else { + manifest_continuity_mode(session, now_ms) + }; + let allowed_forward_window = allowed_manifest_forward_window(session, context, Some(&fetched.body)); + let sequence_relation = classify_manifest_sequence_relation( + previous_highwater, + origin_highwater, + allowed_forward_window, + continuity_mode, + ); + let reject_reason = + manifest_quality_reject_reason(sequence_relation, previous_highwater, origin_highwater, allowed_forward_window); + let score = manifest_origin_quality_score(host_relation, sequence_relation, reject_reason.is_some()); + let should_reset_stall_counter = matches!( + sequence_relation, + HlsManifestSequenceRelation::NoPreviousHighwater + | HlsManifestSequenceRelation::Rebase + | HlsManifestSequenceRelation::Next + | HlsManifestSequenceRelation::PlausibleForward + ); + let should_increment_stall_counter = matches!( + sequence_relation, + HlsManifestSequenceRelation::NoOriginHighwater + | HlsManifestSequenceRelation::Same + | HlsManifestSequenceRelation::ForwardTooFar + | HlsManifestSequenceRelation::Backward + ); + + HlsManifestOriginQuality { + score, + continuity_mode, + host_relation, + sequence_relation, + effective_host, + origin_highwater, + previous_highwater, + allowed_forward_window, + should_increment_stall_counter, + should_reset_stall_counter, + requires_handoff_discontinuity: matches!( + (host_relation, sequence_relation), + (HlsManifestOriginRelation::OtherRedirectHost, _) | (_, HlsManifestSequenceRelation::RolloverCandidate) + ), + reject_reason, + } +} + +fn classify_manifest_sequence_relation( + previous_highwater: Option, + origin_highwater: Option, + allowed_forward_window: Option, + continuity_mode: HlsManifestContinuityMode, +) -> HlsManifestSequenceRelation { + if matches!(continuity_mode, HlsManifestContinuityMode::RebaseAllowed) && origin_highwater.is_some() { + return HlsManifestSequenceRelation::Rebase; + } + let Some(previous_highwater) = previous_highwater else { + return HlsManifestSequenceRelation::NoPreviousHighwater; + }; + let Some(origin_highwater) = origin_highwater else { + return HlsManifestSequenceRelation::NoOriginHighwater; + }; + if origin_highwater == previous_highwater { + return HlsManifestSequenceRelation::Same; + } + if previous_highwater.checked_add(1) == Some(origin_highwater) { + return HlsManifestSequenceRelation::Next; + } + if origin_highwater > previous_highwater { + return if manifest_highwater_delta_within_window( + origin_highwater.saturating_sub(previous_highwater), + allowed_forward_window, + ) { + HlsManifestSequenceRelation::PlausibleForward + } else { + HlsManifestSequenceRelation::ForwardTooFar + }; + } + if origin_highwater_is_within_limit(origin_highwater, allowed_forward_window) { + HlsManifestSequenceRelation::RolloverCandidate + } else { + HlsManifestSequenceRelation::Backward + } +} + +fn manifest_continuity_mode(session: &super::HlsSession, now_ms: u64) -> HlsManifestContinuityMode { + if session.origin_seq_highwater.is_none() { + return HlsManifestContinuityMode::RebaseAllowed; + } + match session.account_binding_protection(now_ms) { + HlsAccountBindingProtection::NoMediaYet | HlsAccountBindingProtection::Expired => { + HlsManifestContinuityMode::RebaseAllowed + } + HlsAccountBindingProtection::HardActive { .. } | HlsAccountBindingProtection::SoftActive { .. } => { + HlsManifestContinuityMode::StrictContinuity + } + } +} + +fn manifest_highwater_delta_within_window(delta: u64, allowed_forward_window: Option) -> bool { + allowed_forward_window.is_none_or(|window| delta <= window.max(1)) +} + +fn manifest_quality_reject_reason( + sequence_relation: HlsManifestSequenceRelation, + previous_highwater: Option, + origin_highwater: Option, + allowed_forward_window: Option, +) -> Option { + match sequence_relation { + HlsManifestSequenceRelation::NoOriginHighwater => { + Some(HlsManifestAcceptanceRejectReason::MissingOriginHighwater) + } + HlsManifestSequenceRelation::ForwardTooFar => Some(HlsManifestAcceptanceRejectReason::ForwardTooFar { + previous: previous_highwater.unwrap_or_default(), + origin: origin_highwater.unwrap_or_default(), + window: allowed_forward_window, + }), + HlsManifestSequenceRelation::Backward => Some(HlsManifestAcceptanceRejectReason::BackwardOutsideRollover { + previous: previous_highwater.unwrap_or_default(), + origin: origin_highwater.unwrap_or_default(), + window: allowed_forward_window, + }), + HlsManifestSequenceRelation::NoPreviousHighwater + | HlsManifestSequenceRelation::Rebase + | HlsManifestSequenceRelation::Same + | HlsManifestSequenceRelation::Next + | HlsManifestSequenceRelation::PlausibleForward + | HlsManifestSequenceRelation::RolloverCandidate => None, + } +} + +fn manifest_origin_quality_score( + host_relation: HlsManifestOriginRelation, + sequence_relation: HlsManifestSequenceRelation, + rejected: bool, +) -> HlsManifestOriginQualityScore { + if rejected { + return HlsManifestOriginQualityScore::Rejected; + } + let same_host = + matches!(host_relation, HlsManifestOriginRelation::Initial | HlsManifestOriginRelation::SameRedirectHost); + match (same_host, sequence_relation) { + (true, HlsManifestSequenceRelation::Next) => HlsManifestOriginQualityScore::SameHostNextSequence, + (true, HlsManifestSequenceRelation::Rebase) => HlsManifestOriginQualityScore::SameHostRebase, + (true, HlsManifestSequenceRelation::NoPreviousHighwater | HlsManifestSequenceRelation::PlausibleForward) => { + HlsManifestOriginQualityScore::SameHostPlausibleForward + } + (true, HlsManifestSequenceRelation::RolloverCandidate) => { + HlsManifestOriginQualityScore::SameHostRolloverCandidate + } + (true, HlsManifestSequenceRelation::Same) => HlsManifestOriginQualityScore::SameHostUnchanged, + (false, HlsManifestSequenceRelation::Next) => HlsManifestOriginQualityScore::OtherHostNextSequence, + (false, HlsManifestSequenceRelation::Rebase) => HlsManifestOriginQualityScore::OtherHostRebaseCandidate, + (false, HlsManifestSequenceRelation::NoPreviousHighwater | HlsManifestSequenceRelation::PlausibleForward) => { + HlsManifestOriginQualityScore::OtherHostPlausibleForward + } + (false, HlsManifestSequenceRelation::RolloverCandidate) => { + HlsManifestOriginQualityScore::OtherHostRolloverCandidate + } + (false, HlsManifestSequenceRelation::Same) => HlsManifestOriginQualityScore::OtherHostUnchanged, + ( + _, + HlsManifestSequenceRelation::NoOriginHighwater + | HlsManifestSequenceRelation::ForwardTooFar + | HlsManifestSequenceRelation::Backward, + ) => HlsManifestOriginQualityScore::Rejected, + } +} + +pub(crate) async fn log_hls_manifest_initial_selected( + context: &HlsOriginManifestFetchContext, + report: &HlsManifestRecoveryCandidateScoreReport, +) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + debug!( + "Manifest '{}' initial selected: host={} media-sequence={} highwater={} score={}", + session_label, + report.quality.effective_host.as_deref().unwrap_or("none"), + report.media_sequence, + format_optional_highwater(report.quality.origin_highwater), + report.quality.score.as_log_value() + ); +} + +async fn log_manifest_recovery_candidate_scored( + context: &HlsOriginManifestFetchContext, + candidate_index: usize, + candidates: usize, + report: &HlsManifestRecoveryCandidateScoreReport, +) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + debug!( + "Manifest '{}' candidate {} of {} scored: host={} media-sequence={} highwater={} score={}", + session_label, + candidate_index + 1, + candidates, + report.quality.effective_host.as_deref().unwrap_or("none"), + report.media_sequence, + format_optional_highwater(report.quality.origin_highwater), + report.quality.score.as_log_value() + ); +} + +async fn log_manifest_recovery_candidate_rejected( + context: &HlsOriginManifestFetchContext, + candidate_index: usize, + candidates: usize, + host: Option<&str>, + highwater: Option, + reason: &HlsManifestRejectLogReason, +) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + debug!( + "Manifest '{}' candidate {} of {} rejected: host={} highwater={} reason={}", + session_label, + candidate_index + 1, + candidates, + host.unwrap_or("none"), + format_optional_highwater(highwater), + reason.status_label() + ); +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum ManifestRecoverySelectionLogPhase { + Recovery, + Burst, +} + +impl ManifestRecoverySelectionLogPhase { + pub(crate) const fn from_candidate_count(candidates: usize) -> Self { + if candidates > 1 { + Self::Burst + } else { + Self::Recovery + } + } + + pub(crate) const fn as_log_label(self) -> &'static str { + match self { + Self::Recovery => "recovery", + Self::Burst => "burst", + } + } +} + +async fn log_manifest_initial_attempt(context: &HlsOriginManifestFetchContext) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + let input_source = context.origin_entry.to_input_source(); + debug!( + "Manifest '{}' attempting URL attempt initial: {} reason=origin-refresh", + session_label, + safe_origin_log_value(input_source.url.as_str()) + ); +} + +async fn log_manifest_recovery_selected( + context: &HlsOriginManifestFetchContext, + candidate_index: usize, + candidates: usize, + report: &HlsManifestRecoveryCandidateScoreReport, +) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + let phase = ManifestRecoverySelectionLogPhase::from_candidate_count(candidates); + debug!( + "Manifest '{}' {} selected candidate {} of {}: host={} media-sequence={} highwater={} score={}", + session_label, + phase.as_log_label(), + candidate_index + 1, + candidates, + report.quality.effective_host.as_deref().unwrap_or("none"), + report.media_sequence, + format_optional_highwater(report.quality.origin_highwater), + report.quality.score.as_log_value() + ); +} + +pub(crate) fn format_optional_highwater(highwater: Option) -> String { + highwater.map_or_else(|| "none".to_string(), |value| value.to_string()) +} + +fn recovery_burst_plan(context: &HlsOriginManifestFetchContext, attempt_index: usize) -> HlsManifestRecoveryBurstPlan { + if attempt_index == 0 { + context.manifest_recovery_burst.level.plan() + } else { + HlsManifestRecoveryBurstLevel::Off.plan() + } +} + +fn request_hls_session_idle_timeout_secs_from_config(app_config: &AppConfig) -> u64 { + app_config + .config + .load() + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()) + .map_or(DEFAULT_HLS_SESSION_IDLE_TIMEOUT_SECS, |hls_cache| hls_cache.session_idle_timeout) + .max(1) +} + +pub(crate) fn allowed_manifest_forward_window( + session: &super::HlsSession, + context: &HlsOriginManifestFetchContext, + body: Option<&str>, +) -> Option { + let timing = body.map(parse_manifest_timing); + let target_duration_secs = timing + .and_then(|timing| timing.target_duration_ms.and_then(|duration_ms| u32::try_from(duration_ms / 1_000).ok())); + origin_highwater_policy_limit( + request_hls_session_idle_timeout_secs_from_config(&context.app_config), + target_duration_secs.or(session.target_duration), + ) +} + +pub(crate) fn origin_highwater_policy_limit( + session_idle_timeout_secs: u64, + target_duration_secs: Option, +) -> Option { + let target_duration_secs = u64::from(target_duration_secs.unwrap_or(DEFAULT_HLS_TARGET_DURATION_SECS)); + if target_duration_secs == 0 { + return None; + } + Some(session_idle_timeout_secs.div_ceil(target_duration_secs)) +} + +fn origin_highwater_is_within_limit(origin_highwater: u64, origin_highwater_limit: Option) -> bool { + origin_highwater_limit.is_some_and(|limit| origin_highwater <= limit) +} + +pub(crate) fn manifest_origin_quality_from_candidate( + candidate: Option<&super::HlsManifestHostSwitchCandidate>, +) -> HlsManifestOriginQuality { + let (effective_host, origin_highwater, score) = + candidate.map_or((None, None, HlsManifestOriginQualityScore::Rejected), |candidate| { + ( + Some(candidate.host.clone()), + candidate.highwater, + manifest_origin_quality_score_from_rank(candidate.quality_score), + ) + }); + HlsManifestOriginQuality { + score, + continuity_mode: HlsManifestContinuityMode::StrictContinuity, + host_relation: HlsManifestOriginRelation::OtherRedirectHost, + sequence_relation: HlsManifestSequenceRelation::NoOriginHighwater, + effective_host, + origin_highwater, + previous_highwater: None, + allowed_forward_window: None, + should_increment_stall_counter: true, + should_reset_stall_counter: false, + requires_handoff_discontinuity: false, + reject_reason: None, + } +} + +fn manifest_origin_quality_score_from_rank(rank: u16) -> HlsManifestOriginQualityScore { + match rank { + 100 => HlsManifestOriginQualityScore::SameHostNextSequence, + 90 => HlsManifestOriginQualityScore::SameHostPlausibleForward, + 85 => HlsManifestOriginQualityScore::SameHostRebase, + 75 => HlsManifestOriginQualityScore::OtherHostNextSequence, + 65 => HlsManifestOriginQualityScore::OtherHostPlausibleForward, + 60 => HlsManifestOriginQualityScore::OtherHostRebaseCandidate, + 50 => HlsManifestOriginQualityScore::SameHostRolloverCandidate, + 35 => HlsManifestOriginQualityScore::OtherHostRolloverCandidate, + 20 => HlsManifestOriginQualityScore::SameHostUnchanged, + 10 => HlsManifestOriginQualityScore::OtherHostUnchanged, + _ => HlsManifestOriginQualityScore::Rejected, + } +} + +pub(crate) fn manifest_host_switch_failure_threshold(session: &super::HlsSession, strip: &StripConfig) -> u32 { + let effective_strip_segments = match strip.mode { + HlsStripMode::Segments => { + u32::try_from(strip.value).unwrap_or(u32::MAX.saturating_sub(HLS_MANIFEST_HOST_SWITCH_BASE_WINDOW_SEGMENTS)) + } + HlsStripMode::Seconds => u32::try_from(session.initial_prefetch_gap_segments) + .unwrap_or(u32::MAX.saturating_sub(HLS_MANIFEST_HOST_SWITCH_BASE_WINDOW_SEGMENTS)), + }; + manifest_host_switch_failure_threshold_for_strip_segments(effective_strip_segments) +} + +pub(crate) fn manifest_host_switch_failure_threshold_for_strip_segments(effective_strip_segments: u32) -> u32 { + HLS_MANIFEST_HOST_SWITCH_BASE_WINDOW_SEGMENTS + .saturating_add(effective_strip_segments) + .saturating_div(2) + .clamp(1, HLS_MANIFEST_HOST_SWITCH_MAX_FAILURE_THRESHOLD) +} + +pub(crate) fn fetched_effective_manifest_host(fetched: &FetchedOriginManifest) -> Option { + if fetched.redirect_host.is_some() { + return fetched.redirect_host.clone(); + } + Url::parse(&fetched.resolved_request_url).ok().and_then(|url| url.host_str().map(str::to_string)) +} + +fn is_hls_retryable_manifest_reject_fetch_error(err: &OriginManifestFetchError) -> bool { + matches!( + err, + OriginManifestFetchError::RetryableStatus(_, _) + | OriginManifestFetchError::Request(_) + | OriginManifestFetchError::Redirect(_) + | OriginManifestFetchError::Timeout + | OriginManifestFetchError::RetryExhausted + ) +} + +pub(crate) fn commit_error_to_fetch_error(err: &HlsManifestCommitError) -> OriginManifestFetchError { + match err { + HlsManifestCommitError::TimelineRejected { .. } | HlsManifestCommitError::RetryCurrentTarget => { + OriginManifestFetchError::RetryExhausted + } + } +} + +fn commit_error_to_retry_reason(err: &HlsManifestCommitError) -> HlsManifestRejectLogReason { + match err { + HlsManifestCommitError::TimelineRejected { reason } => reason.clone(), + HlsManifestCommitError::RetryCurrentTarget => HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + } +} + +pub(crate) fn next_committed_origin_highwater( + current_highwater: Option, + origin_highwater: u64, + sequence_relation: HlsManifestSequenceRelation, +) -> u64 { + match sequence_relation { + HlsManifestSequenceRelation::NoPreviousHighwater + | HlsManifestSequenceRelation::Rebase + | HlsManifestSequenceRelation::Next + | HlsManifestSequenceRelation::PlausibleForward + | HlsManifestSequenceRelation::RolloverCandidate => origin_highwater, + HlsManifestSequenceRelation::NoOriginHighwater + | HlsManifestSequenceRelation::Same + | HlsManifestSequenceRelation::ForwardTooFar + | HlsManifestSequenceRelation::Backward => { + current_highwater.map_or(origin_highwater, |current| current.max(origin_highwater)) + } + } +} + +fn next_retry_delay_ms(retry_policy: &RetryPolicy, attempt_index: usize, retry_after_ms: Option) -> u64 { + retry_after_ms.unwrap_or_else(|| retry_policy.delays_ms.get(attempt_index + 1).copied().unwrap_or_default()) +} + +async fn log_manifest_retry_scheduled( + context: &HlsOriginManifestFetchContext, + attempt_index: usize, + attempts: usize, + delay_ms: u64, + reject_reason: Option<&HlsManifestRejectLogReason>, + fetch_error: Option<&OriginManifestFetchError>, +) { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + let status = manifest_retry_status_label(reject_reason, fetch_error); + warn!( + "Manifest '{}' retry scheduled: status {} attempt {} of {} next_delay_ms={delay_ms}", + session_label, + status, + attempt_index + 1, + attempts + ); +} + +fn manifest_retry_status_label( + reject_reason: Option<&HlsManifestRejectLogReason>, + fetch_error: Option<&OriginManifestFetchError>, +) -> String { + match (reject_reason, fetch_error) { + (Some(reason), Some(err)) => { + format!("{} error={}", reason.status_label(), safe_origin_log_value(format!("{err:?}"))) + } + (Some(reason), None) => reason.status_label(), + (None, Some(err)) => format!("pinned-host-recovery error={}", safe_origin_log_value(format!("{err:?}"))), + (None, None) => "pinned-host-recovery".to_string(), + } +} + +async fn fetch_origin_manifest_once( + entry_url: &Url, + headers: &HeaderMap, + client: &Client, + no_redirect_client: &Client, + use_manual_redirects: bool, + provider_url_index: Option, + origin_manifest_timeout_ms: u64, +) -> Result { + if use_manual_redirects { + fetch_origin_manifest_with_manual_redirects( + entry_url, + headers, + no_redirect_client, + provider_url_index, + origin_manifest_timeout_ms, + ) + .await + } else { + let response = client + .get(entry_url.clone()) + .headers(headers.clone()) + .send() + .await + .map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + response_to_fetched_manifest(response, provider_url_index, entry_url.clone(), origin_manifest_timeout_ms).await + } +} + +async fn fetch_origin_manifest_with_manual_redirects( + entry_url: &Url, + headers: &HeaderMap, + client: &Client, + provider_url_index: Option, + origin_manifest_timeout_ms: u64, +) -> Result { + let mut current_url = entry_url.clone(); + let mut current_headers = headers.clone(); + let mut remaining_redirects = MAX_MANUAL_REDIRECTS; + + loop { + let response = client + .get(current_url.clone()) + .headers(current_headers.clone()) + .send() + .await + .map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + if !response.status().is_redirection() { + return response_to_fetched_manifest( + response, + provider_url_index, + entry_url.clone(), + origin_manifest_timeout_ms, + ) + .await; + } + if remaining_redirects == 0 { + return Err(OriginManifestFetchError::Redirect("too many redirects".to_string())); + } + let response_url = response.url().clone(); + let location = response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| OriginManifestFetchError::Redirect("redirect missing location".to_string()))?; + let next_url = response_url + .join(location) + .or_else(|_| Url::parse(location)) + .map_err(|_| OriginManifestFetchError::Redirect("redirect location invalid".to_string()))?; + + if !same_origin(&response_url, &next_url) { + strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); + } + current_url = next_url; + remaining_redirects = remaining_redirects.saturating_sub(1); + } +} + +async fn fetch_hls_origin_manifest_recovery_direct_target( + context: &HlsOriginManifestFetchContext, + target_url: &Url, + provider_url_index: Option, + reject_reason: Option<&HlsManifestRejectLogReason>, + log_context: ManifestRecoveryAttemptLogContext, +) -> Result { + let session_label = { + let session = context.session.read().await; + safe_session_key(&session.key) + }; + let reason = + reject_reason.map_or_else(|| "pinned-host-recovery".to_string(), HlsManifestRejectLogReason::status_label); + if log_context.candidates > 1 { + debug!( + "Manifest '{}' attempting URL attempt {} of {} candidate {} of {}: {} reason={}", + session_label, + log_context.attempt_index + 1, + log_context.attempts, + log_context.candidate_index + 1, + log_context.candidates, + safe_origin_log_value(target_url.as_str()), + reason + ); + } else { + debug!( + "Manifest '{}' attempting URL attempt {} of {}: {} reason={}", + session_label, + log_context.attempt_index + 1, + log_context.attempts, + safe_origin_log_value(target_url.as_str()), + reason + ); + } + timeout( + Duration::from_millis(context.origin_manifest_timeout_ms.max(1)), + fetch_origin_manifest_once( + target_url, + &context.headers, + &context.client, + &context.no_redirect_client, + context.use_manual_redirects, + provider_url_index, + context.origin_manifest_timeout_ms, + ), + ) + .await + .map_err(|_| OriginManifestFetchError::Timeout)? +} + +#[derive(Debug, Clone, Copy)] +struct ManifestRecoveryAttemptLogContext { + attempt_index: usize, + attempts: usize, + candidate_index: usize, + candidates: usize, +} + +impl ManifestRecoveryAttemptLogContext { + const fn single(attempt_index: usize, attempts: usize) -> Self { + Self { attempt_index, attempts, candidate_index: 0, candidates: 1 } + } +} + +async fn response_to_fetched_manifest( + response: reqwest::Response, + provider_url_index: Option, + resolved_request_url: Url, + origin_manifest_timeout_ms: u64, +) -> Result { + let status = response.status(); + debug!( + "HLS origin manifest response received: request_target={} final_target={} status={}", + safe_origin_log_value(resolved_request_url.as_str()), + safe_origin_log_value(response.url().as_str()), + status.as_u16() + ); + match classify_origin_manifest_status(status) { + OriginManifestStatusClass::Success => { + let final_url = response.url().clone(); + let redirect_host = hls_manifest_redirect_host(&resolved_request_url, &final_url); + let provider_session_headers = extract_hls_provider_session_header_map(response.headers()); + let body = read_origin_manifest_body(response, origin_manifest_timeout_ms).await?; + Ok(FetchedOriginManifest { + body, + final_manifest_url: final_url.to_string(), + resolved_request_url: resolved_request_url.to_string(), + redirect_host, + provider_url_index, + provider_session_headers, + status, + attempts: 1, + }) + } + OriginManifestStatusClass::Retryable => { + Err(OriginManifestFetchError::RetryableStatus(status, retry_after_delay_ms(response.headers()))) + } + OriginManifestStatusClass::PermanentFailure => Err(OriginManifestFetchError::PermanentStatus(status)), + OriginManifestStatusClass::NonRetryableFailure => Err(OriginManifestFetchError::NonRetryableStatus(status)), + } +} + +fn manifest_body_size_after_chunk(current: usize, chunk: usize) -> Option { + current.checked_add(chunk).filter(|size| *size <= MAX_HLS_MANIFEST_BYTES) +} + +async fn read_origin_manifest_body( + response: reqwest::Response, + origin_manifest_timeout_ms: u64, +) -> Result { + if response.content_length().is_some_and(|size| size > MAX_HLS_MANIFEST_BYTES as u64) { + return Err(OriginManifestFetchError::NonRetryableStatus(StatusCode::PAYLOAD_TOO_LARGE)); + } + let read = async move { + let capacity = response.content_length().map_or(0, |size| { + usize::try_from(size.min(MAX_HLS_MANIFEST_BYTES as u64)).unwrap_or(MAX_HLS_MANIFEST_BYTES) + }); + let mut body = Vec::with_capacity(capacity); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| origin_manifest_fetch_error_from_request_error(&err))?; + if manifest_body_size_after_chunk(body.len(), chunk.len()).is_none() { + return Err(OriginManifestFetchError::NonRetryableStatus(StatusCode::PAYLOAD_TOO_LARGE)); + } + body.extend_from_slice(&chunk); + } + String::from_utf8(body).map_err(|_| OriginManifestFetchError::Request("origin manifest is not UTF-8".to_string())) + }; + timeout(Duration::from_millis(origin_manifest_timeout_ms.max(1)), read) + .await + .map_err(|_| OriginManifestFetchError::Timeout)? +} + +pub(crate) fn hls_manifest_redirect_host(resolved_request_url: &Url, final_url: &Url) -> Option { + let final_host = final_url.host_str()?; + (resolved_request_url.host_str() != Some(final_host)).then(|| final_host.to_string()) +} + +pub(crate) fn resolved_hls_manifest_request_url_from_input( + input_source: &InputSource, + provider_url_index: Option, + fallback_url: &Url, +) -> Url { + let fallback = || Url::parse(input_source.url.as_str()).unwrap_or_else(|_| fallback_url.clone()); + let (Some(provider), Some(provider_url_index)) = (input_source.get_provider(), provider_url_index) else { + return fallback(); + }; + match resolve_provider_scheme_url_with_provider_index( + input_source.url.as_str(), + Some(Arc::clone(provider)), + provider_url_index, + ) { + Ok((_provider, resolved_url)) => Url::parse(resolved_url.as_ref()).unwrap_or_else(|err| { + debug!( + "HLS provider URL resolution returned invalid URL: error={} origin={}", + sanitize_sensitive_info(err.to_string().as_str()), + safe_origin_log_value(input_source.url.as_str()) + ); + fallback() + }), + Err(err) => { + debug!( + "HLS provider URL resolution failed: error={} origin={}", + sanitize_sensitive_info(err.to_string().as_str()), + safe_origin_log_value(input_source.url.as_str()) + ); + fallback() + } + } +} + +fn origin_manifest_fetch_error_from_request_error(err: &impl ToString) -> OriginManifestFetchError { + let message = sanitize_sensitive_info(err.to_string().as_str()).to_string(); + let Some(status) = request_failed_status_from_message(&message) else { + return OriginManifestFetchError::Request(message); + }; + match classify_origin_manifest_status(status) { + OriginManifestStatusClass::Success => OriginManifestFetchError::Request(message), + OriginManifestStatusClass::Retryable => OriginManifestFetchError::RetryableStatus(status, None), + OriginManifestStatusClass::PermanentFailure => OriginManifestFetchError::PermanentStatus(status), + OriginManifestStatusClass::NonRetryableFailure => OriginManifestFetchError::NonRetryableStatus(status), + } +} + +fn request_failed_status_from_message(message: &str) -> Option { + let marker = "Request failed ("; + let status_start = message.find(marker)?.checked_add(marker.len())?; + let status_text = message.get(status_start..)?.split(')').next()?; + let status_code = status_text.split_whitespace().next()?.parse::().ok()?; + StatusCode::from_u16(status_code).ok() +} + +fn same_origin(lhs: &Url, rhs: &Url) -> bool { + lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) + && lhs.host_str() == rhs.host_str() + && lhs.port_or_known_default() == rhs.port_or_known_default() +} + +fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HeaderMap) { + super::scrub_hls_origin_headers(headers, None); +} + +pub fn retry_after_delay_ms(headers: &HeaderMap) -> Option { + headers + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .map(|seconds| seconds.saturating_mul(1_000)) +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +pub(crate) async fn refresh_from_live_hls_entrypoint_with_retries( + origin_entry: &LiveHlsOriginEntry, + headers: &HeaderMap, + client: &Client, + no_redirect_client: &Client, + use_manual_redirects: bool, + origin_manifest_timeout_ms: u64, + retry_policy: &RetryPolicy, +) -> Result { + let mut retry_after_delay_ms = None; + let attempts = retry_policy.attempt_count(); + + for attempt_index in 0..attempts { + let delay_ms = retry_after_delay_ms.take().unwrap_or_else(|| { + let jitter = + if retry_policy.jitter_max_ms == 0 { 0 } else { fastrand::u64(0..=retry_policy.jitter_max_ms) }; + retry_policy.delay_for_attempt_ms(attempt_index, jitter).unwrap_or_default() + }); + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + let fetch_result = timeout( + Duration::from_millis(origin_manifest_timeout_ms.max(1)), + fetch_origin_manifest_once( + origin_entry.url(), + headers, + client, + no_redirect_client, + use_manual_redirects, + None, + origin_manifest_timeout_ms, + ), + ) + .await + .map_err(|_| OriginManifestFetchError::Timeout); + + match fetch_result { + Ok(Ok(fetched)) => return Ok(fetched.with_attempts(attempt_index + 1)), + Ok(Err(OriginManifestFetchError::PermanentStatus(status))) => { + return Err(OriginManifestFetchError::PermanentStatus(status)); + } + Ok(Err(OriginManifestFetchError::NonRetryableStatus(status))) => { + return Err(OriginManifestFetchError::NonRetryableStatus(status)); + } + Ok(Err(OriginManifestFetchError::RetryableStatus(status, retry_after_ms))) => { + if attempt_index + 1 == attempts { + return Err(OriginManifestFetchError::RetryableStatus(status, retry_after_ms)); + } + log_origin_refresh_retry_scheduled( + origin_entry, + attempt_index, + next_retry_delay_ms(retry_policy, attempt_index, retry_after_ms), + format!("status={}", status.as_u16()), + ); + retry_after_delay_ms = retry_after_ms; + } + Ok(Err(OriginManifestFetchError::Request(err))) => { + if attempt_index + 1 == attempts { + return Err(OriginManifestFetchError::Request(err)); + } + log_origin_refresh_retry_scheduled( + origin_entry, + attempt_index, + next_retry_delay_ms(retry_policy, attempt_index, None), + format!("error={}", safe_origin_log_value(&err)), + ); + } + Ok(Err(err @ (OriginManifestFetchError::Redirect(_) | OriginManifestFetchError::Timeout))) => { + if attempt_index + 1 == attempts { + return Err(err); + } + log_origin_refresh_retry_scheduled( + origin_entry, + attempt_index, + next_retry_delay_ms(retry_policy, attempt_index, None), + format!("error={}", safe_origin_log_value(format!("{err:?}"))), + ); + } + Ok(Err(OriginManifestFetchError::RetryExhausted)) => return Err(OriginManifestFetchError::RetryExhausted), + Ok(Err(OriginManifestFetchError::ProviderUnavailable(kind))) => { + return Err(OriginManifestFetchError::ProviderUnavailable(kind)); + } + Err(OriginManifestFetchError::Timeout) => { + if attempt_index + 1 == attempts { + return Err(OriginManifestFetchError::Timeout); + } + log_origin_refresh_retry_scheduled( + origin_entry, + attempt_index, + next_retry_delay_ms(retry_policy, attempt_index, None), + "error=timeout", + ); + } + Err(err) => return Err(err), + } + } + + Err(OriginManifestFetchError::RetryExhausted) +} + +#[cfg(test)] +fn log_origin_refresh_retry_scheduled( + origin_entry: &LiveHlsOriginEntry, + attempt_index: usize, + delay_ms: u64, + detail: impl AsRef, +) { + warn!( + "HLS origin manifest refresh retry scheduled: origin_entry={} attempt={} {} delay_ms={delay_ms}", + safe_origin_log_value(origin_entry.url().as_str()), + attempt_index + 1, + detail.as_ref() + ); +} + +#[cfg(test)] +mod tests { + use super::{ + manifest_body_size_after_chunk, origin_manifest_fetch_error_from_request_error, request_failed_status_from_message, + OriginManifestFetchError, MAX_HLS_MANIFEST_BYTES, + }; + use axum::http::StatusCode; + + #[test] + fn request_failed_status_is_extracted_from_global_provider_policy_error() { + assert_eq!( + request_failed_status_from_message( + "Request failed (407 Proxy Authentication Required): provider://demo/live/u/p/1.m3u8", + ), + Some(StatusCode::PROXY_AUTHENTICATION_REQUIRED) + ); + } + + #[test] + fn request_failed_407_maps_to_retryable_manifest_status() { + let err = origin_manifest_fetch_error_from_request_error(& + "Request failed (407 Proxy Authentication Required): provider://demo/live/u/p/1.m3u8", + ); + assert!(matches!( + err, + OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, None) + )); + } + + #[test] + fn request_failed_retryable_statuses_map_to_retryable_manifest_status() { + for (message, expected) in [ + ("Request failed (429 Too Many Requests): http://example.test/live.m3u8", StatusCode::TOO_MANY_REQUESTS), + ( + "Request failed (500 Internal Server Error): http://example.test/live.m3u8", + StatusCode::INTERNAL_SERVER_ERROR, + ), + ] { + let err = origin_manifest_fetch_error_from_request_error(&message); + assert!(matches!(err, OriginManifestFetchError::RetryableStatus(status, None) if status == expected)); + } + } + + #[test] + fn request_failed_404_maps_to_permanent_manifest_status() { + let err = + origin_manifest_fetch_error_from_request_error(&"Request failed (404 Not Found): http://example.test/live.m3u8"); + assert!(matches!(err, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + } + + #[test] + fn transport_error_without_http_status_stays_request_error() { + let err = origin_manifest_fetch_error_from_request_error(&"error sending request for url"); + assert!(matches!(err, OriginManifestFetchError::Request(message) if message == "error sending request for url")); + } + + #[test] + fn manifest_body_limit_rejects_the_first_byte_above_the_limit() { + assert_eq!(manifest_body_size_after_chunk(MAX_HLS_MANIFEST_BYTES - 1, 1), Some(MAX_HLS_MANIFEST_BYTES)); + assert_eq!(manifest_body_size_after_chunk(MAX_HLS_MANIFEST_BYTES, 1), None); + } +} diff --git a/backend/src/api/model/hls_cache/map.rs b/backend/src/api/model/hls_cache/map.rs new file mode 100644 index 000000000..4a4be70e3 --- /dev/null +++ b/backend/src/api/model/hls_cache/map.rs @@ -0,0 +1,193 @@ +use super::{cache::MapCacheKey, ids::ProxySessionId, session::HlsSession, timeline::CacheAccessState}; +use axum::http::StatusCode; +use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use std::{ + fmt, + hash::{Hash, Hasher}, + sync::Arc, +}; + +/// Proxy-visible identifier for one EXT-X-MAP cache resource. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ProxyMapId(pub u64); + +impl From for ProxyMapId { + fn from(value: u64) -> Self { Self(value) } +} + +#[derive(Clone, Eq, PartialEq)] +pub struct OriginMapKey { + pub origin_epoch: u64, + /// Concrete absolute MAP fetch URI resolved against the final manifest URL. + /// + /// This may intentionally contain a provider mirror, redirect target, or CDN + /// host. Do not normalize it back to `provider://` or the original manifest + /// host unless a separate semantic identity is introduced and refetch safety + /// for relative EXT-X-MAP URIs is proven. + pub resolved_origin_uri: String, + pub byte_range: Option, +} + +impl fmt::Debug for OriginMapKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OriginMapKey") + .field("origin_epoch", &self.origin_epoch) + .field("resolved_origin_uri", &"") + .field("byte_range", &self.byte_range) + .finish() + } +} + +impl Hash for OriginMapKey { + fn hash(&self, state: &mut H) { + self.origin_epoch.hash(state); + self.resolved_origin_uri.hash(state); + match self.byte_range { + Some(byte_range) => { + 1_u8.hash(state); + byte_range.length.hash(state); + byte_range.offset.hash(state); + } + None => 0_u8.hash(state), + } + } +} + +/// Volatile concrete origin URL for one EXT-X-MAP download. +/// +/// The URL is resolved against the final fetched manifest URL and may include a redirect/CDN host. Use it only as a +/// fetch target or sanitized diagnostics; stable proxy cache identity is `MapCacheKey`. +#[derive(Clone, Eq, PartialEq)] +pub struct OriginMapFetchRef { + /// Concrete URL used to refetch the MAP object. + /// + /// Keep this aligned with `OriginMapKey::resolved_origin_uri`; relative MAP + /// URIs must remain resolved against the final manifest URL after redirects. + pub resolved_origin_url: String, + pub byte_range: Option, + pub valid_until_ms: Option, +} + +impl OriginMapFetchRef { + pub fn is_valid_at(&self, now_ms: u64) -> bool { + self.valid_until_ms.is_none_or(|valid_until_ms| now_ms <= valid_until_ms) + } +} + +impl fmt::Debug for OriginMapFetchRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OriginMapFetchRef") + .field("resolved_origin_url", &"") + .field("byte_range", &self.byte_range) + .field("valid_until_ms", &self.valid_until_ms) + .finish() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum MapCacheStatus { + Discovered, + Queued { queued_at_ms: u64 }, + Fetching { started_at_ms: u64 }, + Ready { content_length: u64, ready_at_ms: u64 }, + FailedRetryable { failed_at_ms: u64, retry_after_ms: u64 }, + FailedPermanent { failed_at_ms: u64, status: Option }, + Expired, +} + +#[derive(Clone, Eq, PartialEq)] +pub struct MapEntry { + pub proxy_map_id: ProxyMapId, + pub origin_key: OriginMapKey, + pub proxy_file_ext: String, + pub content_type: String, + pub cache_key: MapCacheKey, + pub origin_fetch_ref: Option, + pub byte_range: Option, + pub status: MapCacheStatus, + pub access: Arc, +} + +impl fmt::Debug for MapEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapEntry") + .field("proxy_map_id", &self.proxy_map_id) + .field("origin_key", &self.origin_key) + .field("proxy_file_ext", &self.proxy_file_ext) + .field("content_type", &self.content_type) + .field("cache_key", &self.cache_key) + .field("origin_fetch_ref", &self.origin_fetch_ref) + .field("byte_range", &self.byte_range) + .field("status", &self.status) + .field("active_readers", &self.access.active_readers()) + .field("last_accessed_at_ms", &self.access.last_accessed_at_ms()) + .finish() + } +} + +impl MapEntry { + pub fn default_content_type() -> &'static str { "video/mp4" } + + /// Creates a proxy MAP entry from a concrete origin MAP key. + /// + /// `origin_fetch_ref` intentionally starts from `origin_key.resolved_origin_uri`. + /// Until a separate semantic MAP identity exists, this preserves the final + /// redirect/CDN fetch URL required to reload relative EXT-X-MAP resources. + pub fn new( + proxy_session_id: &ProxySessionId, + proxy_map_id: ProxyMapId, + origin_key: OriginMapKey, + proxy_file_ext: String, + ) -> Self { + Self { + proxy_map_id, + origin_fetch_ref: Some(OriginMapFetchRef { + resolved_origin_url: origin_key.resolved_origin_uri.clone(), + byte_range: origin_key.byte_range, + valid_until_ms: None, + }), + byte_range: origin_key.byte_range, + cache_key: MapCacheKey::new(proxy_session_id.clone(), proxy_map_id, &proxy_file_ext), + content_type: Self::default_content_type().to_string(), + proxy_file_ext, + origin_key, + status: MapCacheStatus::Discovered, + access: Arc::new(CacheAccessState::new()), + } + } +} + +impl HlsSession { + pub fn queue_map_fetch_candidates(&mut self, now_ms: u64) { + if self.is_gc_marked_for_removal() { + return; + } + for map in self.maps.values_mut() { + if matches!(map.status, MapCacheStatus::Discovered) && map.origin_fetch_ref.is_some() { + map.status = MapCacheStatus::Queued { queued_at_ms: now_ms }; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{MapEntry, OriginMapKey, ParsedByteRange, ProxyMapId}; + use crate::api::model::ProxySessionId; + + #[test] + fn map_entry_preserves_concrete_fetch_url_and_byte_range() { + let key = OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "https://cdn.example.net/live/redirected/init.mp4".to_string(), + byte_range: Some(ParsedByteRange { length: 100, offset: 50 }), + }; + + let entry = MapEntry::new(&ProxySessionId("proxy".to_string()), ProxyMapId(7), key.clone(), "mp4".to_string()); + + assert_eq!(entry.origin_key, key); + let fetch_ref = entry.origin_fetch_ref.as_ref().expect("fetch ref"); + assert_eq!(fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/init.mp4"); + assert_eq!(fetch_ref.byte_range, Some(ParsedByteRange { length: 100, offset: 50 })); + } +} diff --git a/backend/src/api/model/hls_cache/map_fetcher.rs b/backend/src/api/model/hls_cache/map_fetcher.rs new file mode 100644 index 000000000..7497f36dd --- /dev/null +++ b/backend/src/api/model/hls_cache/map_fetcher.rs @@ -0,0 +1,720 @@ +use super::{ + begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, finish_hls_origin_account_io, + hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, + HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseStore, HlsBoundAccountAcquireErrorKind, + HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, + HlsSegmentCache, + HlsSessionHandle, MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, +}; +use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use arc_swap::ArcSwap; +use axum::http::HeaderMap; +use futures::{FutureExt, TryStreamExt}; +use log::debug; +use reqwest::Client; +use std::{fmt, io, sync::Arc}; +use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore}; +use tokio_util::io::StreamReader; + +/// Shared context required to schedule EXT-X-MAP origin fetches without holding session locks. +#[derive(Clone)] +pub struct MapFetchContext { + pub session: HlsSessionHandle, + pub segment_cache: Arc, + pub headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub client: Client, + pub no_redirect_client: Client, + pub use_manual_redirects: bool, + pub origin_io: Option, +} + +#[derive(Clone)] +struct MapFetchSnapshot { + proxy_map_id: ProxyMapId, + proxy_map_id_log: String, + cache_key: MapCacheKey, + fetch_ref: OriginMapFetchRef, +} + +struct MapFetchCommit { + content_length: u64, + generation_valid: bool, +} + +#[derive(Clone, Copy)] +struct MapOriginWorkFinish { + generation_valid: bool, + refresh_reservation: bool, +} + +impl fmt::Debug for MapFetchSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapFetchSnapshot") + .field("proxy_map_id", &self.proxy_map_id) + .field("proxy_map_id_log", &self.proxy_map_id_log) + .field("cache_key", &self.cache_key) + .field("fetch_ref", &self.fetch_ref) + .finish() + } +} + +type MapFetchError = HlsOriginResourceFetchError; + +#[derive(Clone)] +struct MapWorkerRuntime { + global_semaphore: Arc, + policy: SegmentFetchPolicy, +} + +impl MapWorkerRuntime { + fn new(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { Self { global_semaphore, policy } } +} + +/// Bounded scheduler for live HLS EXT-X-MAP origin fetches. +pub struct HlsMapWorkerPool { + runtime: ArcSwap, + access_leases: Arc>, +} + +impl HlsMapWorkerPool { + pub fn new(policy: SegmentFetchPolicy) -> Self { + let global_semaphore = Arc::new(Semaphore::new(policy.max_global_segment_fetches)); + Self::with_global_semaphore(policy, global_semaphore) + } + + pub fn with_global_semaphore(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { + Self::with_global_semaphore_and_access_leases( + policy, + global_semaphore, + Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + ) + } + + pub fn with_global_semaphore_and_access_leases( + policy: SegmentFetchPolicy, + global_semaphore: Arc, + access_leases: Arc>, + ) -> Self { + Self { runtime: ArcSwap::from_pointee(MapWorkerRuntime::new(policy, global_semaphore)), access_leases } + } + + pub fn update_config(&self, policy: SegmentFetchPolicy, global_semaphore: Arc) { + self.runtime.store(Arc::new(MapWorkerRuntime::new(policy, global_semaphore))); + } + + pub fn access_leases(&self) -> &Arc> { &self.access_leases } + + pub async fn wake_scheduler(self: &Arc, context: MapFetchContext, now_ms: u64) { + loop { + let runtime = self.runtime.load_full(); + let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { + return; + }; + let Some(snapshot) = self.next_fetch_snapshot(&context, now_ms, &runtime.policy).await else { + drop(permit); + return; + }; + + let worker = Arc::clone(self); + let task_context = context.clone(); + tokio::spawn(async move { + worker.fetch_one_map(task_context, snapshot, runtime.policy.clone(), permit).await; + }); + } + } + + async fn next_fetch_snapshot( + &self, + context: &MapFetchContext, + now_ms: u64, + policy: &SegmentFetchPolicy, + ) -> Option { + let (proxy_session_id, gc_marked_for_removal) = { + let session = context.session.read().await; + (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + }; + if gc_marked_for_removal { + return None; + } + if !self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms) { + let mut session = context.session.write().await; + for map in session.maps.values_mut() { + if matches!(map.status, MapCacheStatus::Queued { .. }) { + map.status = MapCacheStatus::Discovered; + } + } + return None; + } + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() { + return None; + } + if session.active_map_fetches >= policy.max_session_segment_fetches { + return None; + } + + let proxy_map_id = session.maps.iter().find_map(|(proxy_map_id, entry)| { + matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }).then_some(*proxy_map_id) + })?; + + let entry = session.maps.get_mut(&proxy_map_id)?; + let fetch_ref = entry.origin_fetch_ref.clone()?; + if !fetch_ref.is_valid_at(now_ms) { + return None; + } + let cache_key = entry.cache_key.clone(); + entry.status = MapCacheStatus::Fetching { started_at_ms: now_ms }; + session.active_map_fetches = session.active_map_fetches.saturating_add(1); + let proxy_map_id_log = format!("{:06}", proxy_map_id.0); + debug!( + "HLS map fetch started: session={} source=normal resource=map/{}", + super::safe_proxy_session_id(&session.proxy_session_id), + proxy_map_id_log + ); + + Some(MapFetchSnapshot { + proxy_map_id, + proxy_map_id_log, + cache_key, + fetch_ref, + }) + } + + async fn fetch_one_map( + self: Arc, + context: MapFetchContext, + snapshot: MapFetchSnapshot, + policy: SegmentFetchPolicy, + permit: OwnedSemaphorePermit, + ) { + let result = fetch_map_into_cache(&context, &snapshot, &policy).await; + let finished_at_ms = current_time_millis(); + let generation_valid = result.as_ref().map_or(true, |commit| commit.generation_valid); + let fetch_succeeded = result.is_ok(); + let mut response_flag_reason = None; + { + let mut session = context.session.write().await; + session.active_map_fetches = session.active_map_fetches.saturating_sub(1); + if let Some(entry) = session.maps.get_mut(&snapshot.proxy_map_id) { + match result { + Ok(commit) => { + let content_length = commit.content_length; + entry.status = MapCacheStatus::Ready { content_length, ready_at_ms: finished_at_ms }; + debug!( + "HLS map cached: session={} source=normal resource=map/{} content_length={content_length}", + super::safe_proxy_session_id(&session.proxy_session_id), + snapshot.proxy_map_id_log + ); + } + Err(err) => { + if err.retryable_failure() { + entry.status = + MapCacheStatus::FailedRetryable { failed_at_ms: finished_at_ms, retry_after_ms: 1_000 }; + } else { + response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::MapPermanentFailure { + status: err.permanent_status(), + }); + entry.status = + MapCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: err.permanent_status() }; + } + } + } + } + if fetch_succeeded && generation_valid { + let _ = session.render_and_store_manifest(finished_at_ms); + } + } + if let Some(reason) = response_flag_reason { + let marked = self.mark_channel_unavailable_for_session(&context.session, finished_at_ms, reason).await; + if marked > 0 { + debug!( + "HLS access leases marked channel unavailable: session={} marked={marked}", + super::safe_proxy_session_id(&context.session.read().await.proxy_session_id) + ); + } + } + drop(permit); + if generation_valid { + self.schedule_wake(context, finished_at_ms); + } + } + + async fn mark_channel_unavailable_for_session( + &self, + session: &HlsSessionHandle, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> usize { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + self.access_leases + .write() + .await + .mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) + } + + fn schedule_wake(self: &Arc, context: MapFetchContext, now_ms: u64) { + let worker = Arc::clone(self); + tokio::spawn(async move { + worker.wake_scheduler(context, now_ms).await; + }); + } +} + +impl Default for HlsMapWorkerPool { + fn default() -> Self { Self::new(SegmentFetchPolicy::default()) } +} + +async fn fetch_map_into_cache( + context: &MapFetchContext, + snapshot: &MapFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + fetch_map_with_retries_into_cache(context, snapshot, policy).await +} + +struct MapOriginAttemptGuard { + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +} + +async fn prepare_map_origin_attempt( + context: MapFetchContext, + policy: SegmentFetchPolicy, +) -> Result { + let started_generation = start_map_origin_work(&context).await; + let binding = + if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; + let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { + if binding.is_detached() { + let _ = finish_map_origin_work(&context, started_generation).await; + touch_map_origin_account_binding(&context, false).await; + return Err(MapFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); + } + let guard = match begin_hls_origin_account_io_bounded( + origin_io, + &context.session, + binding, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + { + Ok(guard) => guard, + Err(err) => { + let _ = finish_map_origin_work(&context, started_generation).await; + touch_map_origin_account_binding(&context, false).await; + return Err(MapFetchError::ProviderUnavailable(err)); + } + }; + Some((origin_io.clone(), guard)) + } else { + None + }; + Ok(MapOriginAttemptGuard { started_generation, provider_lease }) +} + +async fn finish_map_origin_attempt(context: MapFetchContext, guard: MapOriginAttemptGuard) -> MapOriginWorkFinish { + finish_map_origin_io(&context, guard.started_generation, guard.provider_lease).await +} + +async fn finish_map_origin_io( + context: &MapFetchContext, + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +) -> MapOriginWorkFinish { + let origin_work = finish_map_origin_work(context, started_generation).await; + if let Some((origin_io, guard)) = provider_lease { + finish_hls_origin_account_io( + &origin_io, + &context.session, + guard, + origin_work.generation_valid && origin_work.refresh_reservation, + ) + .await; + touch_map_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) + .await; + } + origin_work +} + +async fn start_map_origin_work(context: &MapFetchContext) -> Option { + context.origin_io.as_ref()?; + let mut session = context.session.write().await; + Some(session.start_origin_work()) +} + +async fn finish_map_origin_work(context: &MapFetchContext, started_generation: Option) -> MapOriginWorkFinish { + let Some(started_generation) = started_generation else { + return MapOriginWorkFinish { generation_valid: true, refresh_reservation: false }; + }; + let mut session = context.session.write().await; + let generation_valid = session.finish_origin_work(started_generation); + let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); + MapOriginWorkFinish { generation_valid, refresh_reservation } +} + +async fn touch_map_origin_account_binding(context: &MapFetchContext, reservation_refreshed: bool) { + let mut session = context.session.write().await; + if let Some(binding) = session.origin_account_binding.as_mut() { + let now_ms = current_time_millis(); + binding.last_origin_io_at_ms = Some(now_ms); + if reservation_refreshed { + binding.last_reservation_refresh_at_ms = Some(now_ms); + } + } +} + +#[allow(clippy::too_many_lines)] +async fn fetch_map_with_retries_into_cache( + context: &MapFetchContext, + snapshot: &MapFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + let headers = build_map_origin_headers( + &context.headers, + &context.origin_provider_session_headers, + snapshot.fetch_ref.byte_range, + )?; + let target = HlsOriginResourceFetchTarget { + kind: HlsResourceFetchKind::Map, + source: HlsResourceFetchSource::Normal, + object_id: snapshot.proxy_map_id_log.clone(), + origin_url: snapshot.fetch_ref.resolved_origin_url.clone(), + headers, + byte_range_expectation: if snapshot.fetch_ref.byte_range.is_some() { + HlsOriginByteRangeExpectation::PartialContent + } else { + HlsOriginByteRangeExpectation::FullObject + }, + }; + let clients = HlsOriginResourceClients { + client: context.client.clone(), + no_redirect_client: context.no_redirect_client.clone(), + use_manual_redirects: context.use_manual_redirects, + }; + let session_log_id = context.session.read().await.proxy_session_id.0.clone(); + let context = context.clone(); + let snapshot = snapshot.clone(); + let commit_policy = policy.clone(); + let policy_for_prepare = policy.clone(); + let prepare_context = context.clone(); + let cleanup_context = context.clone(); + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + clients, + policy, + &session_log_id, + move |_attempt| { + let context = prepare_context.clone(); + let policy = policy_for_prepare.clone(); + async move { prepare_map_origin_attempt(context, policy).await }.boxed() + }, + move |guard| { + let context = cleanup_context.clone(); + async move { + finish_map_origin_attempt(context, guard).await; + } + .boxed() + }, + move |response, _attempt, guard| { + let context = context.clone(); + let snapshot = snapshot.clone(); + let policy = commit_policy.clone(); + async move { + let commit_result = commit_map_response_into_cache(&context, &snapshot, &policy, response).await; + let origin_work = finish_map_origin_attempt(context, guard).await; + commit_result.map(|metadata| MapFetchCommit { + content_length: metadata.size, + generation_valid: origin_work.generation_valid, + }) + } + .boxed() + }, + ) + .await +} + +async fn commit_map_response_into_cache( + context: &MapFetchContext, + snapshot: &MapFetchSnapshot, + policy: &SegmentFetchPolicy, + response: reqwest::Response, +) -> Result { + let deadline = hls_object_body_deadline(policy.origin_segment_timeout_ms); + let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); + context + .segment_cache + .write_temp_and_commit_with_timeout(&snapshot.cache_key, stream_reader, deadline) + .await + .map_err( + |err| { + if err.kind() == io::ErrorKind::TimedOut { + MapFetchError::Timeout + } else { + MapFetchError::cache_commit(&err) + } + }, + ) +} + +fn build_map_origin_headers( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + byte_range: Option, +) -> Result { + build_hls_origin_resource_headers(source_headers, provider_session_headers, byte_range) +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{build_map_origin_headers, HlsMapWorkerPool, MapFetchContext}; + use crate::{ + api::model::{ + HlsAccessLease, HlsAccessLeaseId, HlsPlaybackFamilyKey, HlsSegmentCache, HlsSessionKey, HlsSessionStore, + MapCacheStatus, ProxySessionId, SegmentFetchPolicy, + }, + processing::parser::hls::origin_manifest::{ + parse_origin_media_manifest, OriginManifestParseOutcome, ParsedByteRange, + }, + }; + use axum::http::{header, HeaderMap}; + use std::{sync::Arc, time::Duration}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + fn normal_manifest( + body: &str, + base_url: &str, + ) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, base_url) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + async fn grant_usable_map_access_lease(worker: &HlsMapWorkerPool, proxy_session_id: &ProxySessionId) { + worker.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + HlsAccessLeaseId("map-lease".to_string()), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + 1, + 15_000, + )); + } + + #[test] + fn map_origin_headers_apply_byterange() { + let mut source_headers = HeaderMap::new(); + source_headers.insert(header::AUTHORIZATION, "Bearer secret".parse().expect("header value")); + source_headers.insert(header::COOKIE, "sid=secret".parse().expect("header value")); + source_headers.insert(header::HOST, "origin.example.com".parse().expect("header value")); + source_headers.insert(header::RANGE, "bytes=0-".parse().expect("header value")); + let headers = build_map_origin_headers( + &source_headers, + &HeaderMap::new(), + Some(ParsedByteRange { offset: 10, length: 5 }), + ) + .expect("headers should build"); + + assert_eq!(headers.get(header::RANGE).expect("range"), "bytes=10-14"); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key(header::HOST)); + } + + #[tokio::test] + async fn map_fetch_writes_cache_and_sets_ready_after_commit() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buf = vec![0_u8; 2048]; + let read = socket.read(&mut buf).await.expect("request reads"); + let request = String::from_utf8_lossy(&buf[..read]); + let body = if request.to_ascii_lowercase().contains("range: bytes=10-14") { "map!!" } else { "bad" }; + let status = if body == "map!!" { "206 Partial Content" } else { "200 OK" }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Range: bytes 10-14/20\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.expect("response writes"); + }); + let base_url = format!("http://{addr}/live/index.m3u8"); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"5@10\"\n#EXTINF:4.0,\n1.m4s\n", + &base_url, + ); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path().to_path_buf())); + let worker = Arc::new(HlsMapWorkerPool::new(SegmentFetchPolicy { + retry_jitter_max_ms: 0, + origin_segment_timeout_ms: 1_000, + ..SegmentFetchPolicy::default() + })); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_map_access_lease(&worker, &proxy_session_id).await; + worker + .wake_scheduler( + MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::clone(&cache), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }, + 2, + ) + .await; + + for _ in 0..50 { + if matches!( + session.read().await.maps.values().next().map(|map| &map.status), + Some(MapCacheStatus::Ready { .. }) + ) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + let session_read = session.read().await; + let map = session_read.maps.values().next().expect("map"); + assert!(matches!(map.status, MapCacheStatus::Ready { content_length: 5, .. })); + assert!(cache.metadata(&map.cache_key).await.expect("metadata").is_some()); + server.await.expect("server joins"); + } + + #[tokio::test] + async fn map_fetch_snapshot_uses_concrete_final_map_fetch_url() { + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"5@10\"\n#EXTINF:4.0,\n1.m4s\n", + "https://cdn.example.net/live/redirected/playlist.m3u8", + ); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + + let worker = Arc::new(HlsMapWorkerPool::new(SegmentFetchPolicy::default())); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_map_access_lease(&worker, &proxy_session_id).await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let context = MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path().to_path_buf())), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }; + + let snapshot = worker + .next_fetch_snapshot(&context, 2, &SegmentFetchPolicy::default()) + .await + .expect("map snapshot"); + + assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/init.mp4"); + assert_eq!(snapshot.fetch_ref.byte_range, Some(ParsedByteRange { length: 5, offset: 10 })); + } + + #[tokio::test] + async fn map_fetch_without_usable_access_lease_resets_queue_without_origin_request() { + let base_url = "http://origin.example.com/live/index.m3u8"; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", base_url); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let worker = Arc::new(HlsMapWorkerPool::new(SegmentFetchPolicy::default())); + worker + .wake_scheduler( + MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path().to_path_buf())), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }, + 2, + ) + .await; + + let session = session.read().await; + assert_eq!(session.active_map_fetches, 0); + assert!(matches!(session.maps.values().next().expect("map").status, MapCacheStatus::Discovered)); + } + + #[tokio::test] + async fn map_fetch_is_blocked_for_gc_marked_session() { + let base_url = "http://origin.example.com/live/index.m3u8"; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", base_url); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_map_fetch_candidates(1); + session.mark_for_gc_removal(); + } + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let worker = Arc::new(HlsMapWorkerPool::new(SegmentFetchPolicy::default())); + worker + .wake_scheduler( + MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path().to_path_buf())), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }, + 2, + ) + .await; + + let session = session.read().await; + assert_eq!(session.active_map_fetches, 0); + assert!(matches!(session.maps.values().next().expect("map").status, MapCacheStatus::Queued { .. })); + } +} diff --git a/backend/src/api/model/hls_cache/mod.rs b/backend/src/api/model/hls_cache/mod.rs new file mode 100644 index 000000000..a1e5be3ee --- /dev/null +++ b/backend/src/api/model/hls_cache/mod.rs @@ -0,0 +1,191 @@ +//! HLS shared-session cache: origin fetcher, segment/map/transient/manifest stores, +//! access-lease protocol, GC, observability, and origin-request header policy. +//! +//! # Subsystem map (29 flat files → 7 logical groups) +//! +//! The flat layout below predates the cache state machine's growth; many files +//! span concerns (e.g. `paths.rs`/`deadline.rs`/`ids.rs` are pure infra). +//! The natural cohesion boundaries are: +//! +//! | Group | Files (current name → natural home) | +//! |--------------|------------------------------------------------------------------------------------| +//! | `session` | `session`, `session_store`, `lifecycle`, `ids` (session-token helpers) | +//! | `segment` | `segment_fetcher`, `segment_repair`, `segment_watchdog` | +//! | `map` | `map`, `map_fetcher` | +//! | `manifest` | `manifest_commit`, `manifest_fetch`, `transient`, `transient_fetcher` | +//! | `lease` | `lease` | +//! | `gc` | `gc` | +//! | `infra` | `ids` (token types), `deadline`, `paths`, `headers` (now via `proxy::header_policy`), `backpressure`, `observability`, `timeline`, `qos`, `cache`, `manager`, `refresh`, `renderer`, `response`, `prefetch`, `origin`, `playback`, `resource_fetch` | +//! +//! `proxy/header_policy` already exists as a cross-proxy module (see +//! `api::model::proxy::header_policy::HopByHopHeader`); `headers.rs` is now a +//! thin delegator. The remaining 28 files are intentionally left in place +//! because the cost of moving them (imports, mod.rs churn, public-API re-exports) +//! outweighs the discoverability gain at this commit. Each subsystem +//! migration is a self-contained follow-up PR. +//! +//! Migration order (lowest risk first): +//! 1. `infra` (only `proxy::header_policy` already done; rest stay flat) +//! 2. `lease`, `gc` (each one self-contained today) +//! 3. `map` (one fetcher, one store) +//! 4. `manifest` + `segment` (share transient types; do together) +//! 5. `session` last (largest blast radius; touches lifecycle, store, ids) +//! +//! Until the move lands, treat the table above as the canonical "where do I +//! put this?" map. New files should land in the natural group, not the flat +//! layout. + +mod backpressure; +mod cache; +mod deadline; +mod gc; +mod headers; +mod ids; +mod lease; +mod lifecycle; +mod manifest_commit; +mod manifest_fetch; +mod manager; +mod map; +mod map_fetcher; +mod observability; +mod origin; +mod paths; +mod playback; +mod prefetch; +mod qos; +mod refresh; +mod resource_fetch; +mod renderer; +mod response; +mod segment_fetcher; +mod segment_repair; +mod segment_watchdog; +mod session; +mod session_store; +mod timeline; +mod transient; +mod transient_fetcher; + +pub use self::{ + backpressure::{classify_hls_backpressure, HlsBackpressureState}, + cache::{ + CacheInvalidationOutcome, CachedSegmentMetadata, HlsCacheObjectKey, HlsSegmentCache, MapCacheKey, + SegmentCacheKey, StagedCacheObject, TransientObjectCacheKey, DEFAULT_HLS_CACHE_DURATION_SECS, + DEFAULT_HLS_CACHE_PATH, + }, + deadline::{hls_client_body_send_deadline, hls_object_body_deadline, refresh_hls_client_body_send_deadline}, + gc::{ + build_rewrite_secret_fingerprint, exec_hls_cache_gc, GarbageCollectionPolicy, GarbageCollectionReport, + HlsGarbageCollector, ProtectedSet, + }, + headers::{ + append_hls_provider_session_headers, extract_hls_provider_session_header_map, + extract_hls_provider_session_headers, force_identity_without_range, hls_origin_headers_with_provider_session, + sanitized_hls_origin_headers, scrub_hls_origin_headers, should_remove_hls_origin_header, + }, + ids::{build_proxy_session_id, HlsSessionKey, ProxySessionId}, + lease::{ + new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseChannelUnavailableReason, + HlsAccessLeaseId, HlsAccessLeaseIdleRelease, HlsAccessLeaseLifecycleSnapshot, + HlsAccessLeasePendingDeadline, HlsAccessLeaseResponseFlag, HlsAccessLeaseSessionSnapshot, + HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, + HlsFreshManifestRequiredReason, HlsPlaybackFamilyKey, + }, + lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey, HlsLifecycleManager}, + manifest_commit::{ + hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request, + hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions, + HlsCommittedManifestBody, + }, + manifest_fetch::{ + classify_origin_manifest_status, LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, + RetryPolicy, + }, + manager::{exec_hls_lifecycle, HlsProxyManager}, + map::{MapCacheStatus, MapEntry, OriginMapFetchRef, OriginMapKey, ProxyMapId}, + map_fetcher::{HlsMapWorkerPool, MapFetchContext}, + observability::{ + safe_hls_access_lease_id, safe_origin_log_value, safe_proxy_session_id, safe_session_key, + safe_user_session_token, HlsCacheMetrics, HlsCacheMetricsSnapshot, + }, + origin::{ + acquire_bound_hls_origin_account_handle, begin_hls_origin_account_io, begin_hls_origin_account_io_bounded, + build_hls_origin_session_owner, classify_account_binding_protection, finish_hls_origin_account_io, + finish_hls_origin_io, hls_origin_account_status, origin_account_binding_from_allocation, safe_hls_origin_owner, + HlsAccountBindingProtection, HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, + HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, HlsOriginAccountBinding, + HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginAccountIoLease, + HlsOriginAccountIoLeaseGuard, HlsOriginAccountRebindState, HlsOriginAccountStatus, HlsOriginIoContext, + HlsOriginSource, HlsOriginSourceKind, HlsOriginWorkClass, + }, + paths::{HlsMapFile, HlsSegmentFile, TransientResourceFile}, + playback::{ + validate_hls_access_lease, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLeaseValidationError, + HLS_ACCESS_LEASE_ID_PLACEHOLDER, + }, + prefetch::{ManifestFetchQueueReport, SegmentFetchPriority, SegmentPrefetchQueue}, + qos::{HlsQosMeterInit, HlsQosRegistration, HlsQosRegistry, HlsQosRuntimeConfig}, + refresh::{ + cold_start_retry_after_seconds, maybe_trigger_origin_refresh, trigger_origin_refresh_sync, + HlsManifestCommitRequirement, OriginRefreshRequest, OriginRefreshState, + }, + resource_fetch::{ + build_hls_origin_resource_headers, build_hls_origin_resource_headers_with_client_range, + classify_hls_resource_status, fetch_hls_origin_resource_response, log_hls_resource_attempt_started, + log_hls_resource_attempt_succeeded, log_hls_resource_fetch_failed, log_hls_resource_retry_scheduled, + log_hls_resource_timeout, retry_after_secs_from_ms, run_hls_origin_resource_retry_loop, + run_hls_origin_resource_retry_loop_with_attempt_prepare, HlsOriginByteRangeExpectation, + HlsOriginResourceAttemptCleanupFuture, HlsOriginResourceAttemptPrepareFuture, HlsOriginResourceClients, + HlsOriginResourceCommitFuture, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsResourceFetchAttempt, HlsResourceFetchKind, HlsResourceFetchLogContext, HlsResourceFetchLogStatus, + HlsResourceFetchSource, HlsResourceStatusClass, + }, + renderer::{ + renderer_candidate_window_proxy_seqs, HlsManifestRenderer, RenderError, RenderPolicy, RenderedManifest, + RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, + }, + response::{ + serve_hls_map_cache_outcome, serve_hls_map_cache_response, serve_hls_segment_cache_outcome, + serve_hls_segment_cache_response, serve_hls_transient_object_cache_outcome, + serve_hls_transient_object_cache_response, HlsCacheResponseContext, HlsMediaActivityMarker, + HlsResourceServeFailure, HlsResourceServeOutcome, + }, + segment_fetcher::{HlsSegmentWorkerPool, SegmentDemandFetchOutcome, SegmentFetchContext, SegmentFetchPolicy}, + segment_repair::{ + parse_ffmpeg_warnings, HlsRepairRenderedObjectId, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, WarningCounters, + }, + session::{ + HlsManifestAcceptanceState, HlsManifestHostSwitchCandidate, HlsManifestTemporaryFailureKind, + HlsManifestTemporaryFailureTracker, HlsManifestTemporaryFailureTransition, HlsSegmentFailureObject, + HlsSegmentFailureTracker, HlsSegmentFailureTransition, HlsSession, HlsSessionActivity, HlsSessionMode, + TransientPassthroughReason, + }, + session_store::{ + HlsExpiredSessionMarker, HlsExpiredSessionReason, HlsSessionHandle, HlsSessionStore, + HlsSessionStoreOutcome, + }, + timeline::{ + default_content_type_for_segment_ext, CacheAccessState, OriginSegmentFetchRef, OriginSegmentKey, + is_hls_provisioning_gap_segment, is_hls_provisioning_segment, SegmentCacheStatus, SegmentEntry, + TimelineMapError, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, + HLS_PROVISIONING_SEGMENT_DURATION_MS, + HLS_PROVISIONING_TARGET_DURATION_SECS, + }, + transient::{ + build_transient_resource_id, TransientObjectCacheEntry, TransientObjectCacheStatus, + TransientObjectFetchDecision, TransientObjectRemoval, TransientObjectUnavailableState, + TransientPassthroughState, TransientResourceId, TransientResourceKind, TransientResourceRef, + TransientResourceStore, + }, + transient_fetcher::{ + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, + hls_transient_origin_response, hls_transient_resource_fetch_kind, + is_hls_transient_full_object_cacheable_request, resolve_hls_transient_object_cache_action, + HlsTransientCacheCommitContext, HlsTransientObjectCacheAction, HlsTransientObjectCacheResolution, + HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, + HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, + }, +}; diff --git a/backend/src/api/model/hls_cache/observability.rs b/backend/src/api/model/hls_cache/observability.rs new file mode 100644 index 000000000..067d1a4ee --- /dev/null +++ b/backend/src/api/model/hls_cache/observability.rs @@ -0,0 +1,174 @@ +use super::{HlsAccessLeaseId, HlsSessionKey, ProxySessionId}; +use sha2::{Digest, Sha256}; +use shared::utils::sanitize_sensitive_info; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// In-memory counters for the live HLS cache path. +#[derive(Debug, Default)] +pub struct HlsCacheMetrics { + sessions_created: AtomicU64, + sessions_reused: AtomicU64, + lease_granted: AtomicU64, + lease_denied: AtomicU64, + refresh_started: AtomicU64, + refresh_skipped: AtomicU64, + refresh_completed: AtomicU64, + refresh_retried: AtomicU64, + refresh_failed: AtomicU64, + transient_switches: AtomicU64, + manifest_rendered: AtomicU64, + manifest_render_skipped: AtomicU64, + cache_hits: AtomicU64, + cache_range_hits: AtomicU64, + demand_fetch_started: AtomicU64, + prefetch_queued: AtomicU64, + prefetch_skipped: AtomicU64, + segments_cached: AtomicU64, + gc_runs: AtomicU64, + segments_removed: AtomicU64, + maps_removed: AtomicU64, + secret_marker_mismatch: AtomicU64, + secret_invalidation_deferred: AtomicU64, +} + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct HlsCacheMetricsSnapshot { + pub sessions_created: u64, + pub sessions_reused: u64, + pub lease_granted: u64, + pub lease_denied: u64, + pub refresh_started: u64, + pub refresh_skipped: u64, + pub refresh_completed: u64, + pub refresh_retried: u64, + pub refresh_failed: u64, + pub transient_switches: u64, + pub manifest_rendered: u64, + pub manifest_render_skipped: u64, + pub cache_hits: u64, + pub cache_range_hits: u64, + pub demand_fetch_started: u64, + pub prefetch_queued: u64, + pub prefetch_skipped: u64, + pub segments_cached: u64, + pub gc_runs: u64, + pub segments_removed: u64, + pub maps_removed: u64, + pub secret_marker_mismatch: u64, + pub secret_invalidation_deferred: u64, +} + +impl HlsCacheMetrics { + pub fn record_session_created(&self) { increment(&self.sessions_created, 1); } + pub fn record_session_reused(&self) { increment(&self.sessions_reused, 1); } + pub fn record_lease_granted(&self) { increment(&self.lease_granted, 1); } + pub fn record_lease_denied(&self) { increment(&self.lease_denied, 1); } + pub fn record_refresh_started(&self) { increment(&self.refresh_started, 1); } + pub fn record_refresh_skipped(&self) { increment(&self.refresh_skipped, 1); } + pub fn record_refresh_completed(&self) { increment(&self.refresh_completed, 1); } + pub fn record_refresh_retried(&self) { increment(&self.refresh_retried, 1); } + pub fn record_refresh_failed(&self) { increment(&self.refresh_failed, 1); } + pub fn record_transient_switch(&self) { increment(&self.transient_switches, 1); } + pub fn record_manifest_rendered(&self) { increment(&self.manifest_rendered, 1); } + pub fn record_manifest_render_skipped(&self) { increment(&self.manifest_render_skipped, 1); } + pub fn record_cache_hit(&self) { increment(&self.cache_hits, 1); } + pub fn record_cache_range_hit(&self) { increment(&self.cache_range_hits, 1); } + pub fn record_demand_fetch_started(&self) { increment(&self.demand_fetch_started, 1); } + pub fn record_prefetch_queued(&self, count: usize) { increment(&self.prefetch_queued, count as u64); } + pub fn record_prefetch_skipped(&self, count: usize) { increment(&self.prefetch_skipped, count as u64); } + pub fn record_segment_cached(&self) { increment(&self.segments_cached, 1); } + pub fn record_gc_run(&self) { increment(&self.gc_runs, 1); } + pub fn record_segments_removed(&self, count: usize) { increment(&self.segments_removed, count as u64); } + pub fn record_maps_removed(&self, count: usize) { increment(&self.maps_removed, count as u64); } + pub fn record_secret_marker_mismatch(&self) { increment(&self.secret_marker_mismatch, 1); } + pub fn record_secret_invalidation_deferred(&self) { increment(&self.secret_invalidation_deferred, 1); } + + pub fn snapshot(&self) -> HlsCacheMetricsSnapshot { + HlsCacheMetricsSnapshot { + sessions_created: load(&self.sessions_created), + sessions_reused: load(&self.sessions_reused), + lease_granted: load(&self.lease_granted), + lease_denied: load(&self.lease_denied), + refresh_started: load(&self.refresh_started), + refresh_skipped: load(&self.refresh_skipped), + refresh_completed: load(&self.refresh_completed), + refresh_retried: load(&self.refresh_retried), + refresh_failed: load(&self.refresh_failed), + transient_switches: load(&self.transient_switches), + manifest_rendered: load(&self.manifest_rendered), + manifest_render_skipped: load(&self.manifest_render_skipped), + cache_hits: load(&self.cache_hits), + cache_range_hits: load(&self.cache_range_hits), + demand_fetch_started: load(&self.demand_fetch_started), + prefetch_queued: load(&self.prefetch_queued), + prefetch_skipped: load(&self.prefetch_skipped), + segments_cached: load(&self.segments_cached), + gc_runs: load(&self.gc_runs), + segments_removed: load(&self.segments_removed), + maps_removed: load(&self.maps_removed), + secret_marker_mismatch: load(&self.secret_marker_mismatch), + secret_invalidation_deferred: load(&self.secret_invalidation_deferred), + } + } +} + +pub fn safe_session_key(key: &HlsSessionKey) -> String { short_hash(&key.stable_value()) } + +pub fn safe_hls_access_lease_id(lease_id: &HlsAccessLeaseId) -> String { short_hash(&lease_id.0) } + +pub fn safe_user_session_token(session_token: &str) -> String { short_hash(session_token) } + +pub fn safe_proxy_session_id(proxy_session_id: &ProxySessionId) -> String { short_hash(&proxy_session_id.0) } + +pub fn safe_origin_log_value(value: impl AsRef) -> String { sanitize_sensitive_info(value.as_ref()).into_owned() } + +fn increment(counter: &AtomicU64, count: u64) { counter.fetch_add(count, Ordering::Relaxed); } + +fn load(counter: &AtomicU64) -> u64 { counter.load(Ordering::Relaxed) } + +fn short_hash(value: &str) -> String { + let digest = Sha256::digest(value.as_bytes()); + let value = digest.iter().take(4).fold(0_u32, |value, byte| (value << 8) | u32::from(*byte)); + format!("{value:08x}") +} + +#[cfg(test)] +mod tests { + use super::{safe_origin_log_value, safe_proxy_session_id, safe_session_key, HlsCacheMetrics}; + use crate::api::model::{HlsSessionKey, ProxySessionId}; + + #[test] + fn safe_log_helpers_do_not_emit_credentials_or_full_proxy_session_id() { + let sanitized = safe_origin_log_value("http://user:password@example.com/live/user/password/123.ts"); + assert!(!sanitized.contains("user:password")); + assert!(!sanitized.contains("/user/password/")); + + let proxy_session_id = ProxySessionId("a8f31c9eQ7sLk92pV0mTaw".to_string()); + assert_eq!(safe_proxy_session_id(&proxy_session_id).len(), 8); + assert!(!safe_proxy_session_id(&proxy_session_id).contains("Lk92pV0mTaw")); + } + + #[test] + fn session_key_log_value_is_a_stable_hash_not_raw_key() { + let key = HlsSessionKey::new(1, "12345"); + let safe = safe_session_key(&key); + + assert_eq!(safe.len(), 8); + assert!(!safe.contains("origin.example.com")); + assert_eq!(safe, safe_session_key(&key)); + } + + #[test] + fn metrics_snapshot_reports_recorded_counts() { + let metrics = HlsCacheMetrics::default(); + + metrics.record_session_created(); + metrics.record_prefetch_queued(2); + metrics.record_prefetch_skipped(3); + + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.sessions_created, 1); + assert_eq!(snapshot.prefetch_queued, 2); + assert_eq!(snapshot.prefetch_skipped, 3); + } +} diff --git a/backend/src/api/model/hls_cache/origin.rs b/backend/src/api/model/hls_cache/origin.rs new file mode 100644 index 000000000..0fadab535 --- /dev/null +++ b/backend/src/api/model/hls_cache/origin.rs @@ -0,0 +1,1021 @@ +use super::{safe_proxy_session_id, HlsSession, HlsSessionHandle, HlsSessionKey, ProxySessionId}; +use crate::{ + api::model::{AppState, ConnectionKind, ProviderAllocation, ProviderHandle}, + model::is_input_expired, +}; +use log::debug; +use shared::utils::sanitize_sensitive_info; +use std::{fmt, net::SocketAddr, sync::Arc}; +use std::time::{Duration, Instant}; +use tokio::{ + sync::{Mutex, Notify}, + time::timeout, +}; + +const HLS_ACCOUNT_OVERLAP_FALLBACK_TARGET_DURATION_MS: u64 = 15_000; +const HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK: Duration = Duration::from_millis(25); + +/// Stable source metadata for one shared live-HLS session. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsOriginSource { + pub input_id: u16, + pub input_name: Arc, + pub stream_ref: String, + pub source_kind: HlsOriginSourceKind, +} + +impl HlsOriginSource { + pub fn new( + input_id: u16, + input_name: Arc, + stream_ref: impl Into, + source_kind: HlsOriginSourceKind, + ) -> Self { + Self { input_id, input_name, stream_ref: stream_ref.into(), source_kind } + } + + pub fn from_session_key(key: &HlsSessionKey) -> Self { + Self::new(key.input_id, Arc::from(""), key.stream_ref.clone(), HlsOriginSourceKind::DirectMediaPlaylist) + } + + pub fn session_key(&self) -> HlsSessionKey { HlsSessionKey::new(self.input_id, self.stream_ref.clone()) } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsOriginSourceKind { + XtreamLive, + M3uMediaPlaylist, + DirectMediaPlaylist, +} + +/// Tuliprox input/alias account binding for one shared HLS content session. +#[derive(Clone)] +pub struct HlsOriginAccountBinding { + pub input_name: Arc, + pub account_name: Arc, + pub session_owner: String, + pub pinned_at_ms: u64, + pub last_origin_io_at_ms: Option, + pub last_reservation_refresh_at_ms: Option, + pub binding_mode: HlsOriginAccountBindingMode, + pub generation: u64, +} + +pub struct HlsOriginAccountIoLease { + pub account_name: Arc, + pub session_owner: String, + pub active_io_count: usize, + provider_handle: Option, + acquiring: bool, + notify: Arc, +} + +impl HlsOriginAccountIoLease { + fn acquiring(binding: &HlsOriginAccountBinding) -> Self { + Self { + account_name: Arc::clone(&binding.account_name), + session_owner: binding.session_owner.clone(), + active_io_count: 0, + provider_handle: None, + acquiring: true, + notify: Arc::new(Notify::new()), + } + } + + fn matches_binding(&self, binding: &HlsOriginAccountBinding) -> bool { + self.account_name == binding.account_name && self.session_owner == binding.session_owner + } + + pub const fn is_active_or_acquiring(&self) -> bool { self.active_io_count > 0 || self.acquiring } +} + +impl fmt::Debug for HlsOriginAccountIoLease { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HlsOriginAccountIoLease") + .field("account_name", &self.account_name) + .field("session_owner", &"") + .field("active_io_count", &self.active_io_count) + .field("has_provider_handle", &self.provider_handle.is_some()) + .field("acquiring", &self.acquiring) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone)] +pub struct HlsOriginAccountIoLeaseGuard { + binding: HlsOriginAccountBinding, +} + +impl HlsOriginAccountIoLeaseGuard { + pub fn binding(&self) -> &HlsOriginAccountBinding { &self.binding } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsAccountOverlapTiming { + pub target_duration_ms: u64, + pub hard_active_window_ms: u64, + pub soft_active_window_ms: u64, +} + +impl HlsAccountOverlapTiming { + pub fn from_target_duration_secs(target_duration_secs: Option) -> Self { + let target_duration_ms = target_duration_secs + .map_or(HLS_ACCOUNT_OVERLAP_FALLBACK_TARGET_DURATION_MS, |duration| duration.saturating_mul(1_000)); + Self { + target_duration_ms, + hard_active_window_ms: target_duration_ms, + soft_active_window_ms: target_duration_ms.saturating_mul(2), + } + } + + pub fn reservation_ttl_secs(self) -> u64 { + self.hard_active_window_ms + .saturating_add(self.soft_active_window_ms) + .saturating_add(999) + / 1_000 + + 1 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsAccountBindingProtection { + NoMediaYet, + HardActive { until_ms: u64 }, + SoftActive { reclaim_until_ms: u64 }, + Expired, +} + +impl HlsAccountBindingProtection { + pub const fn as_log_state(self) -> &'static str { + match self { + Self::NoMediaYet => "no-media-yet", + Self::HardActive { .. } => "hard", + Self::SoftActive { .. } => "soft", + Self::Expired => "expired", + } + } +} + +pub fn classify_account_binding_protection( + last_authorized_media_at_ms: Option, + now_ms: u64, + timing: HlsAccountOverlapTiming, +) -> HlsAccountBindingProtection { + let Some(last_media) = last_authorized_media_at_ms else { + return HlsAccountBindingProtection::NoMediaYet; + }; + + let hard_until = last_media.saturating_add(timing.hard_active_window_ms); + if now_ms <= hard_until { + return HlsAccountBindingProtection::HardActive { until_ms: hard_until }; + } + + let reclaim_until = hard_until.saturating_add(timing.soft_active_window_ms); + if now_ms <= reclaim_until { + return HlsAccountBindingProtection::SoftActive { reclaim_until_ms: reclaim_until }; + } + + HlsAccountBindingProtection::Expired +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsOriginAccountBindingMode { + Active, + Speculative { displaced_proxy_session_id: ProxySessionId, reclaim_until_ms: u64 }, + Detached { reason: HlsOriginAccountDetachedReason, detached_at_ms: u64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsOriginAccountDetachedReason { + SoftWindowElapsed, + ReclaimedByOriginalOwner, + PreemptedByHigherPriority, + AccountMissingOrExpired, + IdleNoActiveLease, + Cleanup, +} + +impl HlsOriginAccountDetachedReason { + pub const fn as_log_reason(self) -> &'static str { + match self { + Self::SoftWindowElapsed => "soft-window-elapsed", + Self::ReclaimedByOriginalOwner => "reclaimed-by-original-owner", + Self::PreemptedByHigherPriority => "preempted-by-higher-priority", + Self::AccountMissingOrExpired => "account-missing-or-expired", + Self::IdleNoActiveLease => "idle-no-active-lease", + Self::Cleanup => "cleanup", + } + } +} + +impl HlsOriginAccountBinding { + pub fn new(input_name: Arc, account_name: Arc, proxy_session_id: &ProxySessionId, now_ms: u64) -> Self { + Self { + input_name, + account_name, + session_owner: build_hls_origin_session_owner(proxy_session_id), + pinned_at_ms: now_ms, + last_origin_io_at_ms: None, + last_reservation_refresh_at_ms: None, + binding_mode: HlsOriginAccountBindingMode::Active, + generation: 0, + } + } + + pub fn rebound( + input_name: Arc, + account_name: Arc, + session_owner: String, + generation: u64, + now_ms: u64, + ) -> Self { + Self { + input_name, + account_name, + session_owner, + pinned_at_ms: now_ms, + last_origin_io_at_ms: None, + last_reservation_refresh_at_ms: None, + binding_mode: HlsOriginAccountBindingMode::Active, + generation, + } + } + + pub fn speculative_from( + input_name: Arc, + account_name: Arc, + proxy_session_id: &ProxySessionId, + displaced_proxy_session_id: ProxySessionId, + reclaim_until_ms: u64, + now_ms: u64, + ) -> Self { + let mut binding = Self::new(input_name, account_name, proxy_session_id, now_ms); + binding.binding_mode = + HlsOriginAccountBindingMode::Speculative { displaced_proxy_session_id, reclaim_until_ms }; + binding + } + + pub fn promote_to_active(&mut self) { self.binding_mode = HlsOriginAccountBindingMode::Active; } + + pub fn detach(&mut self, reason: HlsOriginAccountDetachedReason, now_ms: u64) { + self.binding_mode = HlsOriginAccountBindingMode::Detached { reason, detached_at_ms: now_ms }; + self.last_reservation_refresh_at_ms = None; + } + + pub const fn is_active(&self) -> bool { + matches!( + self.binding_mode, + HlsOriginAccountBindingMode::Active | HlsOriginAccountBindingMode::Speculative { .. } + ) + } + + pub const fn is_detached(&self) -> bool { + matches!(self.binding_mode, HlsOriginAccountBindingMode::Detached { .. }) + } +} + +impl fmt::Debug for HlsOriginAccountBinding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HlsOriginAccountBinding") + .field("input_name", &self.input_name) + .field("account_name", &self.account_name) + .field("session_owner", &"") + .field("pinned_at_ms", &self.pinned_at_ms) + .field("last_origin_io_at_ms", &self.last_origin_io_at_ms) + .field("last_reservation_refresh_at_ms", &self.last_reservation_refresh_at_ms) + .field("binding_mode", &self.binding_mode) + .field("generation", &self.generation) + .finish() + } +} + +#[derive(Debug, Clone, Default)] +pub struct HlsOriginAccountRebindState { + pub last_failed_account: Option>, + pub last_rebind_attempt_at_ms: Option, + pub next_rebind_allowed_at_ms: Option, + pub consecutive_rebind_failures: u32, +} + +impl HlsOriginAccountRebindState { + pub fn is_allowed_now(&self, now_ms: u64) -> bool { + self.next_rebind_allowed_at_ms.is_none_or(|next| now_ms >= next) + } + + pub fn mark_attempt_started(&mut self, account_name: Arc, now_ms: u64) { + self.last_failed_account = Some(account_name); + self.last_rebind_attempt_at_ms = Some(now_ms); + self.next_rebind_allowed_at_ms = Some(now_ms.saturating_add(2_000)); + } + + pub fn mark_failed(&mut self, now_ms: u64) { + self.consecutive_rebind_failures = self.consecutive_rebind_failures.saturating_add(1); + self.next_rebind_allowed_at_ms = Some(now_ms.saturating_add(2_000)); + } + + pub fn mark_success(&mut self) { + self.consecutive_rebind_failures = 0; + self.next_rebind_allowed_at_ms = None; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsOriginAccountStatus { + Known, + Missing, + Expired, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsBoundAccountAcquireErrorKind { + Missing, + Expired, + Exhausted, + ReservedForOther, + Detached, + WaitTimedOut, + AcquireTimedOut, + StoreRace, + Unavailable, +} + +impl HlsBoundAccountAcquireErrorKind { + pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired) } + + pub fn is_retryable_resource_failure(self) -> bool { + matches!( + self, + Self::Exhausted | Self::WaitTimedOut | Self::AcquireTimedOut | Self::StoreRace | Self::Unavailable + ) + } + + pub const fn as_log_label(self) -> &'static str { + match self { + Self::Missing => "Missing", + Self::Expired => "Expired", + Self::Exhausted => "Exhausted", + Self::ReservedForOther => "ReservedForOther", + Self::Detached => "Detached", + Self::WaitTimedOut => "WaitTimedOut", + Self::AcquireTimedOut => "AcquireTimedOut", + Self::StoreRace => "StoreRace", + Self::Unavailable => "Unavailable", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsEffectiveOriginAcquirePolicy { + pub connection_kind: ConnectionKind, + pub priority: i8, + pub updated_at_ms: u64, +} + +impl HlsEffectiveOriginAcquirePolicy { + pub const fn new(connection_kind: ConnectionKind, priority: i8, updated_at_ms: u64) -> Self { + Self { connection_kind, priority, updated_at_ms } + } + + pub const fn fallback() -> Self { Self::new(ConnectionKind::Normal, 0, 0) } + + pub const fn with_updated_at(self, updated_at_ms: u64) -> Self { + Self::new(self.connection_kind, self.priority, updated_at_ms) + } + + pub fn has_same_rank_as(self, other: Self) -> bool { + self.connection_kind == other.connection_kind && self.priority == other.priority + } + + pub const fn is_better_than(self, other: Self) -> bool { + match (self.connection_kind, other.connection_kind) { + (ConnectionKind::Normal, ConnectionKind::Soft) => true, + (ConnectionKind::Soft, ConnectionKind::Normal) => false, + _ => self.priority < other.priority, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsEffectiveOriginAcquirePolicyState { + pub current_policy: HlsEffectiveOriginAcquirePolicy, + pub last_supported_at_ms: u64, +} + +impl HlsEffectiveOriginAcquirePolicyState { + pub const fn new(current_policy: HlsEffectiveOriginAcquirePolicy, now_ms: u64) -> Self { + Self { current_policy, last_supported_at_ms: now_ms } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsOriginWorkClass { + ManifestInteractive, + Demand, + Background, +} + +impl HlsOriginWorkClass { + pub const fn as_log_value(self) -> &'static str { + match self { + Self::ManifestInteractive => "manifest_interactive", + Self::Demand => "demand", + Self::Background => "background", + } + } + + pub const fn allows_grace(self) -> bool { matches!(self, Self::ManifestInteractive | Self::Demand) } + + pub const fn allows_speculative_overlap(self) -> bool { matches!(self, Self::ManifestInteractive | Self::Demand) } +} + +#[derive(Clone)] +pub struct HlsOriginIoContext { + pub app_state: Arc, + pub client_addr: SocketAddr, + pub allow_grace: bool, + pub priority: i8, + pub connection_kind: ConnectionKind, + pub reservation_ttl_secs: u64, + pub preacquired_provider_handle: Option>>>, + pub started_generation: Option, +} + +impl HlsOriginIoContext { + pub const fn with_grace(mut self, allow_grace: bool) -> Self { + self.allow_grace = allow_grace; + self + } + + pub fn with_preacquired_provider_handle(mut self, provider_handle: ProviderHandle) -> Self { + self.preacquired_provider_handle = Some(Arc::new(Mutex::new(Some(provider_handle)))); + self + } + + pub async fn take_preacquired_provider_handle(&self) -> Option { + let handle = self.preacquired_provider_handle.as_ref()?; + handle.lock().await.take() + } +} + +pub fn build_hls_origin_session_owner(proxy_session_id: &ProxySessionId) -> String { + format!("hls-cache:{}", proxy_session_id.0.replace('|', "")) +} + +pub fn hls_origin_account_status( + app_state: &Arc, + binding: &HlsOriginAccountBinding, +) -> HlsOriginAccountStatus { + let Some(provider_config) = app_state.active_provider.find_provider_config(&binding.account_name) else { + return HlsOriginAccountStatus::Missing; + }; + if !app_state.active_provider.is_provider_for_input(&binding.account_name, &binding.input_name) { + return HlsOriginAccountStatus::Missing; + } + if is_input_expired(provider_config.exp_date()) { + return HlsOriginAccountStatus::Expired; + } + HlsOriginAccountStatus::Known +} + +pub async fn acquire_bound_hls_origin_account_handle( + app_state: &Arc, + binding: &HlsOriginAccountBinding, + client_addr: &SocketAddr, + allow_grace: bool, + priority: i8, + connection_kind: ConnectionKind, +) -> Result { + if binding.is_detached() { + return Err(HlsBoundAccountAcquireErrorKind::Detached); + } + match hls_origin_account_status(app_state, binding) { + HlsOriginAccountStatus::Missing => return Err(HlsBoundAccountAcquireErrorKind::Missing), + HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), + HlsOriginAccountStatus::Known => {} + } + if app_state + .active_provider + .is_provider_reserved_for_other_session(&binding.account_name, Some(&binding.session_owner)) + .await + { + return Err(HlsBoundAccountAcquireErrorKind::ReservedForOther); + } + let handle = app_state + .active_provider + .acquire_exact_connection_with_grace_for_session( + &binding.account_name, + client_addr, + allow_grace, + priority, + connection_kind, + Some(&binding.session_owner), + ) + .await; + if let Some(handle) = handle { + return Ok(handle); + } + let kind = if app_state.active_provider.is_exhausted(&binding.account_name).await { + HlsBoundAccountAcquireErrorKind::Exhausted + } else { + HlsBoundAccountAcquireErrorKind::Unavailable + }; + debug!( + "HLS bound origin account unavailable: account={} owner={} reason={kind:?}", + sanitize_sensitive_info(binding.account_name.as_ref()), + sanitize_sensitive_info(&binding.session_owner) + ); + Err(kind) +} + +pub async fn begin_hls_origin_account_io( + origin_io: &HlsOriginIoContext, + session: &HlsSessionHandle, + binding: &HlsOriginAccountBinding, +) -> Result { + begin_hls_origin_account_io_inner(origin_io, session, binding, None).await +} + +pub async fn begin_hls_origin_account_io_bounded( + origin_io: &HlsOriginIoContext, + session: &HlsSessionHandle, + binding: &HlsOriginAccountBinding, + acquire_timeout: Duration, +) -> Result { + begin_hls_origin_account_io_inner(origin_io, session, binding, Some(acquire_timeout)).await +} + +async fn begin_hls_origin_account_io_inner( + origin_io: &HlsOriginIoContext, + session: &HlsSessionHandle, + binding: &HlsOriginAccountBinding, + acquire_timeout: Option, +) -> Result { + if binding.is_detached() { + return Err(HlsBoundAccountAcquireErrorKind::Detached); + } + + if matches!( + reserve_hls_origin_account_io_slot(session, binding, acquire_timeout).await?, + HlsOriginAccountIoSlot::Joined + ) { + if let Some(unused_handle) = origin_io.take_preacquired_provider_handle().await { + origin_io.app_state.connection_manager.release_provider_handle(Some(unused_handle)).await; + } + return Ok(HlsOriginAccountIoLeaseGuard { binding: binding.clone() }); + } + + let acquired_handle = if let Some(handle) = origin_io.take_preacquired_provider_handle().await { + Ok(handle) + } else { + let acquire = acquire_bound_hls_origin_account_handle( + &origin_io.app_state, + binding, + &origin_io.client_addr, + origin_io.allow_grace, + origin_io.priority, + origin_io.connection_kind, + ); + if let Some(acquire_timeout) = acquire_timeout { + if let Ok(result) = timeout(acquire_timeout, acquire).await { + result + } else { + clear_pending_hls_origin_account_io_lease(session, binding).await; + return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); + } + } else { + acquire.await + } + }; + + match acquired_handle { + Ok(handle) => store_acquired_hls_origin_account_io_handle(origin_io, session, binding, handle).await, + Err(err) => { + clear_pending_hls_origin_account_io_lease(session, binding).await; + Err(err) + } + } +} + +async fn reserve_hls_origin_account_io_slot( + session: &HlsSessionHandle, + binding: &HlsOriginAccountBinding, + acquire_timeout: Option, +) -> Result { + let wait_deadline = acquire_timeout.map(|duration| Instant::now() + duration); + loop { + match try_reserve_hls_origin_account_io_slot(&mut *session.write().await, binding) { + HlsOriginAccountIoReserveOutcome::Joined => { + return Ok(HlsOriginAccountIoSlot::Joined); + } + HlsOriginAccountIoReserveOutcome::Acquire => { + return Ok(HlsOriginAccountIoSlot::Acquire); + } + HlsOriginAccountIoReserveOutcome::Wait(notify) => { + if let Some(deadline) = wait_deadline { + let now = Instant::now(); + if now >= deadline { + return Err(HlsBoundAccountAcquireErrorKind::WaitTimedOut); + } + let wait_for = deadline + .saturating_duration_since(now) + .min(HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK); + tokio::select! { + () = notify.notified() => {} + () = tokio::time::sleep(wait_for) => {} + } + } else { + tokio::select! { + () = notify.notified() => {} + () = tokio::time::sleep(HLS_ORIGIN_ACCOUNT_IO_WAIT_RECHECK) => {} + } + } + } + HlsOriginAccountIoReserveOutcome::Unavailable => { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } + HlsOriginAccountIoReserveOutcome::ReservedForOther => { + return Err(HlsBoundAccountAcquireErrorKind::ReservedForOther); + } + } + } +} + +enum HlsOriginAccountIoSlot { + Joined, + Acquire, +} + +enum HlsOriginAccountIoReserveOutcome { + Joined, + Acquire, + Wait(Arc), + ReservedForOther, + Unavailable, +} + +fn try_reserve_hls_origin_account_io_slot( + session: &mut HlsSession, + binding: &HlsOriginAccountBinding, +) -> HlsOriginAccountIoReserveOutcome { + if session.is_gc_marked_for_removal() { + return HlsOriginAccountIoReserveOutcome::Unavailable; + } + + match session.origin_account_io_lease.as_mut() { + Some(lease) if lease.matches_binding(binding) && lease.provider_handle.is_some() => { + lease.active_io_count = lease.active_io_count.saturating_add(1); + HlsOriginAccountIoReserveOutcome::Joined + } + Some(lease) if lease.matches_binding(binding) && lease.acquiring => { + HlsOriginAccountIoReserveOutcome::Wait(Arc::clone(&lease.notify)) + } + Some(lease) if lease.is_active_or_acquiring() => HlsOriginAccountIoReserveOutcome::ReservedForOther, + Some(_) => { + session.origin_account_io_lease = None; + HlsOriginAccountIoReserveOutcome::Acquire + } + None => { + session.origin_account_io_lease = Some(HlsOriginAccountIoLease::acquiring(binding)); + HlsOriginAccountIoReserveOutcome::Acquire + } + } +} + +async fn store_acquired_hls_origin_account_io_handle( + origin_io: &HlsOriginIoContext, + session: &HlsSessionHandle, + binding: &HlsOriginAccountBinding, + handle: ProviderHandle, +) -> Result { + origin_io + .app_state + .active_provider + .refresh_provider_reservation(&binding.account_name, &binding.session_owner, origin_io.reservation_ttl_secs) + .await; + + let mut release_handle = None; + let mut notify_waiters = None; + { + let mut session = session.write().await; + if let Some(lease) = session + .origin_account_io_lease + .as_mut() + .filter(|lease| lease.matches_binding(binding) && lease.acquiring && lease.provider_handle.is_none()) + { + lease.provider_handle = Some(handle); + lease.acquiring = false; + lease.active_io_count = 1; + notify_waiters = Some(Arc::clone(&lease.notify)); + } else { + release_handle = Some(handle); + } + } + if let Some(notify) = notify_waiters { + notify.notify_waiters(); + } + if let Some(handle) = release_handle { + origin_io.app_state.connection_manager.release_provider_handle(Some(handle)).await; + origin_io.app_state.active_provider.clear_provider_reservation(&binding.session_owner).await; + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + Ok(HlsOriginAccountIoLeaseGuard { binding: binding.clone() }) +} + +async fn clear_pending_hls_origin_account_io_lease(session: &HlsSessionHandle, binding: &HlsOriginAccountBinding) { + let notify_waiters = { + let mut session = session.write().await; + let notify = session + .origin_account_io_lease + .as_ref() + .filter(|lease| lease.matches_binding(binding) && lease.acquiring) + .map(|lease| Arc::clone(&lease.notify)); + if notify.is_some() { + session.origin_account_io_lease = None; + } + notify + }; + if let Some(notify) = notify_waiters { + notify.notify_waiters(); + } +} + +pub async fn finish_hls_origin_io( + app_state: &Arc, + binding: &HlsOriginAccountBinding, + provider_handle: Option, + reservation_ttl_secs: u64, +) { + app_state.connection_manager.release_provider_handle(provider_handle).await; + app_state + .active_provider + .refresh_provider_reservation(&binding.account_name, &binding.session_owner, reservation_ttl_secs) + .await; +} + +pub async fn finish_hls_origin_account_io( + origin_io: &HlsOriginIoContext, + session: &HlsSessionHandle, + guard: HlsOriginAccountIoLeaseGuard, + refresh_reservation: bool, +) { + let binding = guard.binding; + let mut provider_handle_to_release = None; + let mut should_refresh_reservation = false; + let mut should_clear_reservation = false; + { + let mut session = session.write().await; + if let Some(lease) = session + .origin_account_io_lease + .as_mut() + .filter(|lease| lease.matches_binding(&binding) && lease.active_io_count > 0) + { + lease.active_io_count = lease.active_io_count.saturating_sub(1); + if lease.active_io_count == 0 { + provider_handle_to_release = lease.provider_handle.take(); + session.origin_account_io_lease = None; + if refresh_reservation { + should_refresh_reservation = true; + } else { + should_clear_reservation = true; + } + } + } + + if let Some(current) = session.origin_account_binding.as_mut().filter(|current| { + current.is_active() + && current.account_name == binding.account_name + && current.session_owner == binding.session_owner + }) { + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + current.last_origin_io_at_ms = Some(now_ms); + if should_refresh_reservation { + current.last_reservation_refresh_at_ms = Some(now_ms); + } + } + } + + if let Some(provider_handle) = provider_handle_to_release { + if should_refresh_reservation { + finish_hls_origin_io(&origin_io.app_state, &binding, Some(provider_handle), origin_io.reservation_ttl_secs) + .await; + } else { + origin_io.app_state.connection_manager.release_provider_handle(Some(provider_handle)).await; + if should_clear_reservation { + origin_io.app_state.active_provider.clear_provider_reservation(&binding.session_owner).await; + } + } + } +} + +pub fn origin_account_binding_from_allocation( + input_name: Arc, + proxy_session_id: &ProxySessionId, + allocation: &ProviderAllocation, + now_ms: u64, +) -> Option { + let account_name = allocation.get_provider_name()?; + Some(HlsOriginAccountBinding::new(input_name, account_name, proxy_session_id, now_ms)) +} + +pub fn safe_hls_origin_owner(proxy_session_id: &ProxySessionId) -> String { + format!("hls-cache:{}", safe_proxy_session_id(proxy_session_id)) +} + +#[cfg(test)] +mod tests { + use super::{ + build_hls_origin_session_owner, classify_account_binding_protection, HlsAccountBindingProtection, + HlsAccountOverlapTiming, HlsBoundAccountAcquireErrorKind, HlsOriginAccountBinding, HlsOriginAccountBindingMode, + HlsOriginAccountRebindState, HlsOriginSource, HlsOriginSourceKind, + }; + use crate::api::model::{build_proxy_session_id, HlsSessionKey, ProxySessionId}; + use std::sync::Arc; + + #[test] + fn hls_origin_session_owner_contains_no_reservation_family_separator() { + let owner = build_hls_origin_session_owner(&ProxySessionId("abc|def".to_string())); + + assert_eq!(owner, "hls-cache:abcdef"); + assert!(!owner.contains('|')); + } + + #[test] + fn origin_source_session_key_uses_input_and_stream_ref() { + let direct = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::XtreamLive); + + assert_eq!(direct.session_key().stable_value(), "input:7|hls|80510"); + } + + #[test] + fn account_overlap_timing_uses_target_duration_or_fallback() { + let timing = HlsAccountOverlapTiming::from_target_duration_secs(Some(12)); + assert_eq!(timing.target_duration_ms, 12_000); + assert_eq!(timing.hard_active_window_ms, 12_000); + assert_eq!(timing.soft_active_window_ms, 24_000); + assert_eq!(timing.reservation_ttl_secs(), 37); + + let fallback = HlsAccountOverlapTiming::from_target_duration_secs(None); + assert_eq!(fallback.target_duration_ms, 15_000); + assert_eq!(fallback.hard_active_window_ms, 15_000); + assert_eq!(fallback.soft_active_window_ms, 30_000); + assert_eq!(fallback.reservation_ttl_secs(), 46); + } + + #[test] + fn account_binding_protection_classifies_hard_soft_and_expired() { + let timing = HlsAccountOverlapTiming::from_target_duration_secs(Some(10)); + + assert_eq!( + classify_account_binding_protection(Some(1_000), 5_000, timing), + HlsAccountBindingProtection::HardActive { until_ms: 11_000 } + ); + assert_eq!( + classify_account_binding_protection(Some(1_000), 20_000, timing), + HlsAccountBindingProtection::SoftActive { reclaim_until_ms: 31_000 } + ); + assert_eq!( + classify_account_binding_protection(Some(1_000), 32_000, timing), + HlsAccountBindingProtection::Expired + ); + assert_eq!(classify_account_binding_protection(None, 1_000, timing), HlsAccountBindingProtection::NoMediaYet); + } + + #[test] + fn speculative_binding_records_displaced_session_and_can_promote() { + let proxy_session_id = ProxySessionId("new-session".to_string()); + let displaced = ProxySessionId("old-session".to_string()); + let mut binding = HlsOriginAccountBinding::speculative_from( + Arc::from("input"), + Arc::from("account"), + &proxy_session_id, + displaced.clone(), + 10_000, + 1_000, + ); + + assert_eq!( + binding.binding_mode, + HlsOriginAccountBindingMode::Speculative { + displaced_proxy_session_id: displaced, + reclaim_until_ms: 10_000, + } + ); + + binding.promote_to_active(); + assert_eq!(binding.binding_mode, HlsOriginAccountBindingMode::Active); + } + + #[test] + fn provider_failover_mirror_urls_do_not_affect_session_or_proxy_identity() { + let mirror_a = "http://mirror-a.example.com"; + let mirror_b = "http://mirror-b.example.com"; + let failover = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist); + let direct = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist); + let secret = b"rewrite-secret"; + + assert_eq!(failover.session_key(), direct.session_key()); + assert_eq!( + build_proxy_session_id(&failover.session_key(), secret), + build_proxy_session_id(&direct.session_key(), secret) + ); + assert!(!failover.session_key().stable_value().contains(mirror_a)); + assert!(!failover.session_key().stable_value().contains(mirror_b)); + assert!(!failover.session_key().stable_value().contains("provider://")); + } + + #[test] + fn origin_account_binding_debug_excludes_origin_and_provider_url_fields() { + let binding = HlsOriginAccountBinding::new( + Arc::from("cdn-dev"), + Arc::from("cdn-dev-alias"), + &ProxySessionId("proxy-session".to_string()), + 100, + ); + let debug = format!("{binding:?}"); + + assert!(debug.contains("cdn-dev-alias")); + assert!(!debug.contains("provider://")); + assert!(!debug.contains("origin_url")); + assert!(!debug.contains("origin_password")); + assert!(!debug.contains("mirror")); + assert!(!debug.contains("proxy-session")); + } + + #[test] + fn session_key_fallback_source_uses_key_identity_only() { + let key = HlsSessionKey::new(9, "stable-content-id"); + let source = HlsOriginSource::from_session_key(&key); + + assert_eq!(source.session_key(), key); + } + + #[test] + fn bound_account_acquire_error_rebind_policy_matches_concept() { + assert!(HlsBoundAccountAcquireErrorKind::Missing.allows_rebind()); + assert!(HlsBoundAccountAcquireErrorKind::Expired.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::Exhausted.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::ReservedForOther.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::Detached.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::WaitTimedOut.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::AcquireTimedOut.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::StoreRace.allows_rebind()); + assert!(!HlsBoundAccountAcquireErrorKind::Unavailable.allows_rebind()); + } + + #[test] + fn bound_account_acquire_error_resource_retry_policy_matches_handoff_semantics() { + assert!(!HlsBoundAccountAcquireErrorKind::Missing.is_retryable_resource_failure()); + assert!(!HlsBoundAccountAcquireErrorKind::Expired.is_retryable_resource_failure()); + assert!(!HlsBoundAccountAcquireErrorKind::ReservedForOther.is_retryable_resource_failure()); + assert!(!HlsBoundAccountAcquireErrorKind::Detached.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::Exhausted.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::WaitTimedOut.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::AcquireTimedOut.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::StoreRace.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::Unavailable.is_retryable_resource_failure()); + } + + #[test] + fn rebound_binding_keeps_owner_and_increments_generation() { + let original = HlsOriginAccountBinding::new( + Arc::from("cdn-dev"), + Arc::from("old-account"), + &ProxySessionId("proxy-session".to_string()), + 100, + ); + + let rebound = HlsOriginAccountBinding::rebound( + Arc::clone(&original.input_name), + Arc::from("new-account"), + original.session_owner.clone(), + original.generation.saturating_add(1), + 200, + ); + + assert_eq!(rebound.session_owner, original.session_owner); + assert_eq!(rebound.generation, original.generation + 1); + assert_eq!(rebound.account_name.as_ref(), "new-account"); + assert_eq!(rebound.pinned_at_ms, 200); + assert_eq!(rebound.last_origin_io_at_ms, None); + assert_eq!(rebound.last_reservation_refresh_at_ms, None); + } + + #[test] + fn origin_account_rebind_state_enforces_backoff() { + let mut state = HlsOriginAccountRebindState::default(); + + assert!(state.is_allowed_now(1_000)); + state.mark_attempt_started(Arc::from("old-account"), 1_000); + assert!(!state.is_allowed_now(1_001)); + state.mark_failed(1_000); + + assert!(!state.is_allowed_now(2_999)); + assert!(state.is_allowed_now(3_000)); + assert_eq!(state.consecutive_rebind_failures, 1); + + state.mark_success(); + assert!(state.is_allowed_now(3_001)); + assert_eq!(state.consecutive_rebind_failures, 0); + assert_eq!(state.next_rebind_allowed_at_ms, None); + } +} diff --git a/backend/src/api/model/hls_cache/paths.rs b/backend/src/api/model/hls_cache/paths.rs new file mode 100644 index 000000000..96af310e7 --- /dev/null +++ b/backend/src/api/model/hls_cache/paths.rs @@ -0,0 +1,139 @@ +use super::TransientResourceId; + +const MIN_PROXY_ID_DIGITS: usize = 6; +const SEGMENT_EXTENSIONS: &[&str] = &["ts", "mp4", "m4s", "m4v"]; +const MAP_EXTENSIONS: &[&str] = &["mp4", "m4s", "m4v"]; +const TRANSIENT_RESOURCE_EXTENSIONS: &[&str] = &["ts", "mpegts", "mp4", "m4s", "m4v", "key"]; + +/// Parsed normal timeline segment file from `/hls/shared/live/{id}/{segment_file}`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsSegmentFile { + pub proxy_seq: u64, + pub extension: String, +} + +/// Parsed EXT-X-MAP file from `/hls/shared/live/{id}/map/{map_file}`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsMapFile { + pub proxy_map_id: u64, + pub extension: String, +} + +/// Parsed transient passthrough resource from `/hls/shared/live/{id}/r/{resource_file}`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransientResourceFile { + pub resource_id: TransientResourceId, + pub extension: String, +} + +impl HlsSegmentFile { + pub fn parse(file_name: &str) -> Option { + let (id, extension) = parse_numeric_file(file_name, SEGMENT_EXTENSIONS)?; + Some(Self { proxy_seq: id, extension: extension.to_string() }) + } +} + +impl HlsMapFile { + pub fn parse(file_name: &str) -> Option { + let (id, extension) = parse_numeric_file(file_name, MAP_EXTENSIONS)?; + Some(Self { proxy_map_id: id, extension: extension.to_string() }) + } +} + +impl TransientResourceFile { + pub fn parse(file_name: &str) -> Option { + if file_name.contains('/') || file_name.contains("://") { + return None; + } + let (resource_id, extension) = file_name.rsplit_once('.')?; + if resource_id.is_empty() + || extension.is_empty() + || !is_opaque_resource_id(resource_id) + || !is_safe_transient_extension(extension) + { + return None; + } + Some(Self { resource_id: TransientResourceId(resource_id.to_string()), extension: extension.to_string() }) + } +} + +fn parse_numeric_file<'a>(file_name: &'a str, allowed_extensions: &[&str]) -> Option<(u64, &'a str)> { + let (digits, extension) = file_name.rsplit_once('.')?; + if digits.len() < MIN_PROXY_ID_DIGITS || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + if !allowed_extensions.contains(&extension) { + return None; + } + Some((digits.parse().ok()?, extension)) +} + +fn is_opaque_resource_id(resource_id: &str) -> bool { + resource_id.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +fn is_safe_transient_extension(extension: &str) -> bool { + extension.bytes().all(|byte| byte.is_ascii_alphanumeric()) && TRANSIENT_RESOURCE_EXTENSIONS.contains(&extension) +} + +#[cfg(test)] +mod tests { + use super::{HlsMapFile, HlsSegmentFile, TransientResourceFile}; + + #[test] + fn segment_file_parser_accepts_supported_extensions_and_minimum_width() { + assert_eq!(HlsSegmentFile::parse("000123.ts").expect("valid segment").proxy_seq, 123); + assert_eq!(HlsSegmentFile::parse("000123.mp4").expect("valid segment").extension, "mp4"); + assert_eq!(HlsSegmentFile::parse("000123.m4s").expect("valid segment").extension, "m4s"); + assert_eq!(HlsSegmentFile::parse("000123.m4v").expect("valid segment").extension, "m4v"); + assert_eq!(HlsSegmentFile::parse("000000123.ts").expect("valid segment").proxy_seq, 123); + } + + #[test] + fn map_file_parser_accepts_supported_extensions_and_minimum_width() { + assert_eq!(HlsMapFile::parse("000001.mp4").expect("valid map").proxy_map_id, 1); + assert_eq!(HlsMapFile::parse("000001.m4s").expect("valid map").extension, "m4s"); + assert_eq!(HlsMapFile::parse("000001.m4v").expect("valid map").extension, "m4v"); + assert_eq!(HlsMapFile::parse("000000001.mp4").expect("valid map").proxy_map_id, 1); + } + + #[test] + fn segment_file_parser_rejects_invalid_names() { + for file_name in ["123.ts", "abc123.ts", "000123.exe", "000123"] { + assert!(HlsSegmentFile::parse(file_name).is_none(), "{file_name} should be rejected"); + } + } + + #[test] + fn map_file_parser_rejects_invalid_names() { + for file_name in ["123.ts", "abc123.ts", "000123.exe", "000123"] { + assert!(HlsMapFile::parse(file_name).is_none(), "{file_name} should be rejected"); + } + } + + #[test] + fn transient_resource_parser_accepts_opaque_resource_ids() { + assert_eq!( + TransientResourceFile::parse("f91ac2.ts").expect("valid transient resource").resource_id.0, + "f91ac2" + ); + assert_eq!(TransientResourceFile::parse("resource_1.m4s").expect("valid transient resource").extension, "m4s"); + assert_eq!(TransientResourceFile::parse("abc-123.key").expect("valid transient resource").extension, "key"); + } + + #[test] + fn transient_resource_parser_rejects_missing_parts_and_origin_urls() { + for file_name in [ + ".ts", + "f91ac2", + "f91ac2.", + "f91ac2.tar.gz", + "f91ac2.m4s.tmp", + "f91ac2.exe", + "http://origin/seg.ts", + "provider://demo/seg.ts", + ] { + assert!(TransientResourceFile::parse(file_name).is_none(), "{file_name} should be rejected"); + } + } +} diff --git a/backend/src/api/model/hls_cache/playback.rs b/backend/src/api/model/hls_cache/playback.rs new file mode 100644 index 000000000..5f5941654 --- /dev/null +++ b/backend/src/api/model/hls_cache/playback.rs @@ -0,0 +1,153 @@ +use super::{ + safe_hls_access_lease_id, safe_proxy_session_id, safe_user_session_token, HlsAccessLease, HlsAccessLeaseId, + HlsPlaybackFamilyKey, ProxySessionId, +}; +use crate::{ + api::{ + api_utils::{connection_priority_for_kind, resolve_playback_request_admission, EvictionReentryGuard}, + model::AppState, + }, + auth::Fingerprint, +}; +use log::warn; +use shared::model::{PlaylistItemType, UserConnectionPermission}; +use std::sync::Arc; + +/// Placeholder stored in shared HLS manifests before per-user access lease IDs are inserted. +pub const HLS_ACCESS_LEASE_ID_PLACEHOLDER: &str = "__hls_access_lease_id__"; + +/// Validated user context restored from a server-side HLS access lease. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsAccessContext { + pub username: String, + pub user_session_token: String, + pub proxy_session_id: ProxySessionId, + pub input_id: u16, + pub stream_ref: String, + pub virtual_id: u32, + pub lease_id: HlsAccessLeaseId, + pub family_key: HlsPlaybackFamilyKey, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessLeaseValidationError { + InvalidLease, + SessionMismatch, + UserSessionMissing, + AdmissionDenied, + Expired, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsAccessAdmissionMode { + ManifestPrepare, + ResourceAccess, +} + +pub async fn validate_hls_access_lease( + app_state: &Arc, + fingerprint: &Fingerprint, + path_proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + admission_mode: HlsAccessAdmissionMode, +) -> Result { + let Some(lease) = app_state.hls_proxy.access_lease(lease_id, path_proxy_session_id, now_ms).await else { + return Err(HlsAccessLeaseValidationError::Expired); + }; + if lease.state == super::HlsAccessLeaseState::Denied { + return Err(HlsAccessLeaseValidationError::AdmissionDenied); + } + validate_hls_access_lease_admission(app_state, fingerprint, lease, admission_mode).await +} + +async fn validate_hls_access_lease_admission( + app_state: &Arc, + fingerprint: &Fingerprint, + lease: HlsAccessLease, + admission_mode: HlsAccessAdmissionMode, +) -> Result { + let Some(user) = app_state.app_config.get_user_credentials(&lease.username) else { + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} reason=user_missing", + safe_hls_access_lease_id(&lease.lease_id), + safe_proxy_session_id(&lease.proxy_session_id), + safe_user_session_token(&lease.user_session_token) + ); + return Err(HlsAccessLeaseValidationError::UserSessionMissing); + }; + let Some(user_session) = + app_state.active_users.get_and_update_user_session(&lease.username, &lease.user_session_token).await + else { + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} reason=session_missing", + safe_hls_access_lease_id(&lease.lease_id), + safe_proxy_session_id(&lease.proxy_session_id), + safe_user_session_token(&lease.user_session_token) + ); + return Err(HlsAccessLeaseValidationError::UserSessionMissing); + }; + + let (admission, _, _) = resolve_playback_request_admission( + app_state, + &user, + fingerprint, + PlaylistItemType::LiveHls, + Some(&user_session), + &lease.user_session_token, + true, + EvictionReentryGuard::Session(&lease.user_session_token), + admission_mode == HlsAccessAdmissionMode::ManifestPrepare, + false, + ) + .await; + if admission.permission == UserConnectionPermission::Exhausted + || (admission.permission == UserConnectionPermission::GracePeriod && admission.kind.is_none()) + { + app_state.hls_proxy.deny_access_lease(&lease.lease_id).await; + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} reason=admission_denied", + safe_hls_access_lease_id(&lease.lease_id), + safe_proxy_session_id(&lease.proxy_session_id), + safe_user_session_token(&lease.user_session_token) + ); + return Err(HlsAccessLeaseValidationError::AdmissionDenied); + } + + let Some(connection_kind) = app_state + .active_users + .refresh_session_connection_kind_for_origin_policy( + &lease.username, + user.max_connections, + user.soft_connections, + &lease.user_session_token, + ) + .await + .or(admission.kind) + else { + app_state.hls_proxy.deny_access_lease(&lease.lease_id).await; + warn!( + "HLS access lease rejected: lease={} proxy_session={} session={} reason=origin_policy_missing", + safe_hls_access_lease_id(&lease.lease_id), + safe_proxy_session_id(&lease.proxy_session_id), + safe_user_session_token(&lease.user_session_token) + ); + return Err(HlsAccessLeaseValidationError::AdmissionDenied); + }; + let priority = connection_priority_for_kind(&user, connection_kind); + let _ = + app_state.hls_proxy.update_access_lease_origin_acquire_policy(&lease.lease_id, connection_kind, priority).await; + + app_state.active_users.touch_http_activity(&lease.username, &lease.user_session_token, &fingerprint.addr).await; + + Ok(HlsAccessContext { + username: lease.username, + user_session_token: lease.user_session_token, + proxy_session_id: lease.proxy_session_id, + input_id: lease.input_id, + stream_ref: lease.stream_ref, + virtual_id: lease.virtual_id, + lease_id: lease.lease_id, + family_key: lease.family_key, + }) +} diff --git a/backend/src/api/model/hls_cache/prefetch.rs b/backend/src/api/model/hls_cache/prefetch.rs new file mode 100644 index 000000000..f36e8c6a7 --- /dev/null +++ b/backend/src/api/model/hls_cache/prefetch.rs @@ -0,0 +1,407 @@ +use super::{HlsSession, SegmentCacheStatus}; +use std::collections::{BTreeSet, HashMap}; + +/// Fetch priority for live HLS segment origin requests. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum SegmentFetchPriority { + Demand, + RenderWindow, + Prefetch, +} + +impl SegmentFetchPriority { + pub const fn rank(self) -> u8 { + match self { + Self::Demand => 0, + Self::RenderWindow => 1, + Self::Prefetch => 2, + } + } + + pub const fn is_higher_priority_than(self, other: Self) -> bool { self.rank() < other.rank() } +} + +/// Session-local deduplicating priority queue for segment fetch candidates. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct SegmentPrefetchQueue { + max_prefetch_depth: usize, + demand: BTreeSet, + render_window: BTreeSet, + prefetch: BTreeSet, + queued: HashMap, +} + +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +pub struct ManifestFetchQueueReport { + pub render_window_queued: usize, + pub prefetch_queued: usize, + pub prefetch_skipped: usize, +} + +impl SegmentPrefetchQueue { + pub fn new(max_prefetch_depth: usize) -> Self { + Self { + max_prefetch_depth, + demand: BTreeSet::new(), + render_window: BTreeSet::new(), + prefetch: BTreeSet::new(), + queued: HashMap::new(), + } + } + + pub fn max_prefetch_depth(&self) -> usize { self.max_prefetch_depth } + + pub fn set_max_prefetch_depth(&mut self, max_prefetch_depth: usize) -> Vec { + self.max_prefetch_depth = max_prefetch_depth; + let mut removed = Vec::new(); + while self.prefetch.len() > self.max_prefetch_depth { + let Some(proxy_seq) = self.prefetch.pop_last() else { + break; + }; + self.queued.remove(&proxy_seq); + removed.push(proxy_seq); + } + removed + } + + pub fn enqueue(&mut self, proxy_seq: u64, priority: SegmentFetchPriority) -> bool { + if priority == SegmentFetchPriority::Prefetch && !self.queued.contains_key(&proxy_seq) { + let protected_prefetch_capacity = self.max_prefetch_depth; + if self.prefetch.len() >= protected_prefetch_capacity { + return false; + } + } + + match self.queued.get(&proxy_seq).copied() { + Some(existing) if priority.is_higher_priority_than(existing) => { + self.remove_from_priority(proxy_seq, existing); + self.insert_into_priority(proxy_seq, priority); + self.queued.insert(proxy_seq, priority); + true + } + Some(_) => false, + None => { + self.insert_into_priority(proxy_seq, priority); + self.queued.insert(proxy_seq, priority); + true + } + } + } + + pub fn pop_next(&mut self) -> Option<(u64, SegmentFetchPriority)> { + if let Some(proxy_seq) = self.demand.pop_first() { + self.queued.remove(&proxy_seq); + return Some((proxy_seq, SegmentFetchPriority::Demand)); + } + if let Some(proxy_seq) = self.render_window.pop_first() { + self.queued.remove(&proxy_seq); + return Some((proxy_seq, SegmentFetchPriority::RenderWindow)); + } + if let Some(proxy_seq) = self.prefetch.pop_first() { + self.queued.remove(&proxy_seq); + return Some((proxy_seq, SegmentFetchPriority::Prefetch)); + } + None + } + + pub fn remove(&mut self, proxy_seq: u64) -> Option { + let priority = self.queued.remove(&proxy_seq)?; + self.remove_from_priority(proxy_seq, priority); + Some(priority) + } + + pub fn contains(&self, proxy_seq: u64) -> bool { self.queued.contains_key(&proxy_seq) } + + pub fn len(&self) -> usize { self.queued.len() } + + pub fn is_empty(&self) -> bool { self.queued.is_empty() } + + pub fn prefetch_len(&self) -> usize { self.prefetch.len() } + + pub fn proxy_seqs(&self) -> Vec { self.queued.keys().copied().collect() } + + fn insert_into_priority(&mut self, proxy_seq: u64, priority: SegmentFetchPriority) { + match priority { + SegmentFetchPriority::Demand => { + self.demand.insert(proxy_seq); + } + SegmentFetchPriority::RenderWindow => { + self.render_window.insert(proxy_seq); + } + SegmentFetchPriority::Prefetch => { + self.prefetch.insert(proxy_seq); + } + } + } + + fn remove_from_priority(&mut self, proxy_seq: u64, priority: SegmentFetchPriority) { + match priority { + SegmentFetchPriority::Demand => { + self.demand.remove(&proxy_seq); + } + SegmentFetchPriority::RenderWindow => { + self.render_window.remove(&proxy_seq); + } + SegmentFetchPriority::Prefetch => { + self.prefetch.remove(&proxy_seq); + } + } + } +} + +impl Default for SegmentPrefetchQueue { + fn default() -> Self { Self::new(6) } +} + +impl HlsSession { + pub fn queue_segment_fetch_candidate( + &mut self, + proxy_seq: u64, + priority: SegmentFetchPriority, + now_ms: u64, + ) -> bool { + if self.is_gc_marked_for_removal() { + return false; + } + let Some(entry) = self.segments.get_mut(&proxy_seq) else { + return false; + }; + if entry.origin_fetch_ref.is_none() { + return false; + } + + match entry.status { + SegmentCacheStatus::Discovered => { + if self.segment_prefetch_queue.enqueue(proxy_seq, priority) { + entry.status = SegmentCacheStatus::Queued { priority, queued_at_ms: now_ms }; + return true; + } + false + } + SegmentCacheStatus::Queued { priority: existing_priority, queued_at_ms } => { + if priority.is_higher_priority_than(existing_priority) { + self.segment_prefetch_queue.enqueue(proxy_seq, priority); + entry.status = SegmentCacheStatus::Queued { priority, queued_at_ms }; + return true; + } + false + } + SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::Ready { .. } + | SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::FailedPermanent { .. } + | SegmentCacheStatus::Expired => false, + } + } + + pub fn queue_manifest_prefetch_candidates(&mut self, now_ms: u64) { + let _ = self.queue_manifest_fetch_candidates(now_ms, true); + } + + pub fn queue_manifest_fetch_candidates(&mut self, now_ms: u64, allow_prefetch: bool) -> ManifestFetchQueueReport { + let mut report = ManifestFetchQueueReport::default(); + if self.is_gc_marked_for_removal() { + return report; + } + let Some(tail_proxy_seq) = self.publishable_origin_tail_proxy_seq else { + return report; + }; + let Some(head_proxy_seq) = self.publishable_origin_head_proxy_seq else { + return report; + }; + if tail_proxy_seq < head_proxy_seq { + return report; + } + let known_sequences = self + .segments + .keys() + .copied() + .filter(|proxy_seq| *proxy_seq >= head_proxy_seq && *proxy_seq <= tail_proxy_seq) + .collect::>(); + if known_sequences.is_empty() { + return report; + } + + let tail_index = + known_sequences.len().saturating_sub(1).saturating_sub(self.initial_prefetch_gap_segments); + let render_window_len = known_sequences.len().min(6).min(tail_index.saturating_add(1)); + let render_start_index = tail_index.saturating_add(1).saturating_sub(render_window_len); + + for proxy_seq in &known_sequences[render_start_index..=tail_index] { + if self.queue_segment_fetch_candidate(*proxy_seq, SegmentFetchPriority::RenderWindow, now_ms) { + report.render_window_queued = report.render_window_queued.saturating_add(1); + } + } + + let prefetch_start = tail_index.saturating_add(1); + if !allow_prefetch { + report.prefetch_skipped = known_sequences + .iter() + .skip(prefetch_start) + .take(self.segment_prefetch_queue.max_prefetch_depth()) + .filter(|proxy_seq| { + self.segments + .get(proxy_seq) + .is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Discovered)) + }) + .count(); + return report; + } + for proxy_seq in + known_sequences.iter().skip(prefetch_start).take(self.segment_prefetch_queue.max_prefetch_depth()) + { + if self.queue_segment_fetch_candidate(*proxy_seq, SegmentFetchPriority::Prefetch, now_ms) { + report.prefetch_queued = report.prefetch_queued.saturating_add(1); + } + } + report + } + + pub fn configure_segment_prefetch_queue(&mut self, max_prefetch_depth: usize) { + for proxy_seq in self.segment_prefetch_queue.set_max_prefetch_depth(max_prefetch_depth) { + if let Some(entry) = self.segments.get_mut(&proxy_seq) { + if matches!( + entry.status, + SegmentCacheStatus::Queued { + priority: SegmentFetchPriority::Prefetch, + .. + } + ) { + entry.status = SegmentCacheStatus::Discovered; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{SegmentFetchPriority, SegmentPrefetchQueue}; + use crate::{ + api::model::{HlsSession, HlsSessionKey, SegmentCacheStatus}, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + }; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn session() -> HlsSession { HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0) } + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + #[test] + fn queue_deduplicates_proxy_sequence() { + let mut queue = SegmentPrefetchQueue::new(6); + + assert!(queue.enqueue(10, SegmentFetchPriority::Prefetch)); + assert!(!queue.enqueue(10, SegmentFetchPriority::Prefetch)); + + assert_eq!(queue.len(), 1); + } + + #[test] + fn demand_upgrades_existing_prefetch_or_render_window_item() { + let mut queue = SegmentPrefetchQueue::new(6); + + assert!(queue.enqueue(10, SegmentFetchPriority::Prefetch)); + assert!(queue.enqueue(10, SegmentFetchPriority::Demand)); + + assert_eq!(queue.pop_next(), Some((10, SegmentFetchPriority::Demand))); + assert!(queue.is_empty()); + } + + #[test] + fn pop_order_is_priority_then_chronological() { + let mut queue = SegmentPrefetchQueue::new(6); + + assert!(queue.enqueue(30, SegmentFetchPriority::Prefetch)); + assert!(queue.enqueue(20, SegmentFetchPriority::Demand)); + assert!(queue.enqueue(10, SegmentFetchPriority::Demand)); + assert!(queue.enqueue(40, SegmentFetchPriority::RenderWindow)); + + assert_eq!(queue.pop_next(), Some((10, SegmentFetchPriority::Demand))); + assert_eq!(queue.pop_next(), Some((20, SegmentFetchPriority::Demand))); + assert_eq!(queue.pop_next(), Some((40, SegmentFetchPriority::RenderWindow))); + assert_eq!(queue.pop_next(), Some((30, SegmentFetchPriority::Prefetch))); + } + + #[test] + fn max_prefetch_depth_limits_prefetch_only() { + let mut queue = SegmentPrefetchQueue::new(1); + + assert!(queue.enqueue(30, SegmentFetchPriority::Prefetch)); + assert!(!queue.enqueue(31, SegmentFetchPriority::Prefetch)); + assert!(queue.enqueue(10, SegmentFetchPriority::Demand)); + assert!(queue.enqueue(20, SegmentFetchPriority::RenderWindow)); + + assert_eq!(queue.len(), 3); + } + + #[test] + fn max_prefetch_depth_shrink_reports_removed_prefetch_sequences() { + let mut queue = SegmentPrefetchQueue::new(3); + + assert!(queue.enqueue(30, SegmentFetchPriority::Prefetch)); + assert!(queue.enqueue(31, SegmentFetchPriority::Prefetch)); + assert!(queue.enqueue(32, SegmentFetchPriority::Prefetch)); + + assert_eq!(queue.set_max_prefetch_depth(1), vec![32, 31]); + assert_eq!(queue.pop_next(), Some((30, SegmentFetchPriority::Prefetch))); + assert!(queue.is_empty()); + } + + #[test] + fn manifest_fetch_candidates_skip_prefetch_when_backpressure_disallows_it() { + let mut session = session(); + session.configure_segment_prefetch_queue(6); + session.initial_prefetch_gap_segments = 2; + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n#EXTINF:4.0,\n103.ts\n#EXTINF:4.0,\n104.ts\n#EXTINF:4.0,\n105.ts\n", + )) + .expect("manifest maps"); + + let report = session.queue_manifest_fetch_candidates(10, false); + + assert_eq!(report.render_window_queued, 4); + assert_eq!(report.prefetch_queued, 0); + assert_eq!(report.prefetch_skipped, 2); + assert!(session.initial_manifest_commit_work_pending()); + assert!(session.segments.values().all(|segment| !matches!( + segment.status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, .. } + ))); + } + + #[test] + fn shrinking_session_prefetch_depth_reverts_trimmed_segment_status() { + let mut session = session(); + session.configure_segment_prefetch_queue(3); + session.initial_prefetch_gap_segments = 3; + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n#EXTINF:4.0,\n103.ts\n#EXTINF:4.0,\n104.ts\n#EXTINF:4.0,\n105.ts\n", + )) + .expect("manifest maps"); + + let report = session.queue_manifest_fetch_candidates(10, true); + assert_eq!(report.prefetch_queued, 3); + + session.configure_segment_prefetch_queue(1); + + assert!(matches!( + session.segments.get(&4).expect("trimmed segment").status, + SegmentCacheStatus::Discovered + )); + assert!(matches!( + session.segments.get(&5).expect("trimmed segment").status, + SegmentCacheStatus::Discovered + )); + } +} diff --git a/backend/src/api/model/hls_cache/qos.rs b/backend/src/api/model/hls_cache/qos.rs new file mode 100644 index 000000000..625782571 --- /dev/null +++ b/backend/src/api/model/hls_cache/qos.rs @@ -0,0 +1,172 @@ +use super::{HlsAccessLeaseId, ProxySessionId}; +use crate::{ + api::model::StreamMeterHandle, + model::AppConfig, +}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; + +/// Runtime decision for Shared-HLS `QoS` hooks derived from global reverse proxy settings. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HlsQosRuntimeConfig { + pub live_metering_enabled: bool, +} + +impl HlsQosRuntimeConfig { + pub fn from_app_config(app_config: &AppConfig) -> Self { + let config = app_config.config.load(); + let Some(reverse_proxy) = config.reverse_proxy.as_ref() else { + return Self::default(); + }; + Self { + live_metering_enabled: reverse_proxy.stream.as_ref().is_some_and(|stream| stream.metrics_enabled), + } + } +} + +pub struct HlsQosMeterInit { + pub meter_uid: u32, + pub meter: Arc, +} + +pub struct HlsQosRegistration { + pub meter_uid: u32, + pub meter: Option>, + pub register_meter: Option>, + pub emit_connect_record: bool, +} + +#[derive(Clone)] +struct HlsAccessLeaseQosState { + proxy_session_id: ProxySessionId, + meter_uid: u32, + meter: Option>, +} + +impl HlsAccessLeaseQosState { + fn registration(&self, emit_connect_record: bool, register_meter: Option>) -> HlsQosRegistration { + HlsQosRegistration { + meter_uid: self.meter_uid, + meter: self.meter.clone(), + register_meter, + emit_connect_record, + } + } +} + +#[derive(Default)] +pub struct HlsQosRegistry { + states: RwLock>, +} + +impl HlsQosRegistry { + pub async fn ensure_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + _now_ms: u64, + meter_init: Option, + ) -> HlsQosRegistration { + let mut states = self.states.write().await; + if let Some(state) = states.get(lease_id) { + return state.registration(false, None); + } + + let (meter_uid, meter) = meter_init.map_or((0, None), |init| (init.meter_uid, Some(init.meter))); + let register_meter = meter.clone(); + let state = HlsAccessLeaseQosState { + proxy_session_id: proxy_session_id.clone(), + meter_uid, + meter, + }; + let registration = state.registration(true, register_meter); + states.insert(lease_id.clone(), state); + registration + } + + pub async fn meter_for_access_lease(&self, lease_id: &HlsAccessLeaseId) -> Option> { + self.states.read().await.get(lease_id).and_then(|state| state.meter.clone()) + } + + pub async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) -> bool { + self.states.write().await.remove(lease_id).is_some() + } + + pub async fn remove_access_leases(&self, lease_ids: &[HlsAccessLeaseId]) -> usize { + let mut states = self.states.write().await; + lease_ids + .iter() + .filter(|lease_id| states.remove(*lease_id).is_some()) + .count() + } + + pub async fn remove_proxy_session_state(&self, proxy_session_id: &ProxySessionId) -> usize { + let mut states = self.states.write().await; + let before = states.len(); + states.retain(|_, state| &state.proxy_session_id != proxy_session_id); + before.saturating_sub(states.len()) + } + + pub async fn clear(&self) -> usize { + let mut states = self.states.write().await; + let removed = states.len(); + states.clear(); + removed + } + + pub async fn len(&self) -> usize { self.states.read().await.len() } + + pub async fn is_empty(&self) -> bool { self.states.read().await.is_empty() } +} + +#[cfg(test)] +mod tests { + use super::{HlsQosMeterInit, HlsQosRegistry}; + use crate::api::model::{EventManager, StreamMeterHandle}; + use crate::api::model::{HlsAccessLeaseId, ProxySessionId}; + use std::sync::Arc; + + #[tokio::test] + async fn qos_registration_emits_connect_once_per_access_lease() { + let registry = HlsQosRegistry::default(); + let event_manager = Arc::new(EventManager::new()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("session-a".to_string()); + let meter = Arc::new(StreamMeterHandle::new(7, Arc::downgrade(&event_manager))); + + let first = registry + .ensure_access_lease( + &lease_id, + &proxy_session_id, + 100, + Some(HlsQosMeterInit { meter_uid: 7, meter: Arc::clone(&meter) }), + ) + .await; + let second = registry.ensure_access_lease(&lease_id, &proxy_session_id, 200, None).await; + + assert!(first.emit_connect_record); + assert_eq!(first.meter_uid, 7); + assert!(first.register_meter.is_some()); + assert!(!second.emit_connect_record); + assert_eq!(second.meter_uid, 7); + assert!(second.register_meter.is_none()); + assert!(Arc::ptr_eq(second.meter.as_ref().expect("meter must be retained"), &meter)); + } + + #[tokio::test] + async fn qos_cleanup_removes_only_matching_session_state() { + let registry = HlsQosRegistry::default(); + let session_a = ProxySessionId("session-a".to_string()); + let session_b = ProxySessionId("session-b".to_string()); + let lease_a = HlsAccessLeaseId("lease-a".to_string()); + let lease_b = HlsAccessLeaseId("lease-b".to_string()); + + registry.ensure_access_lease(&lease_a, &session_a, 100, None).await; + registry.ensure_access_lease(&lease_b, &session_b, 100, None).await; + + assert_eq!(registry.remove_proxy_session_state(&session_a).await, 1); + assert_eq!(registry.len().await, 1); + assert!(registry.meter_for_access_lease(&lease_a).await.is_none()); + assert!(registry.remove_access_lease(&lease_b).await); + } +} diff --git a/backend/src/api/model/hls_cache/refresh.rs b/backend/src/api/model/hls_cache/refresh.rs new file mode 100644 index 000000000..80333bce1 --- /dev/null +++ b/backend/src/api/model/hls_cache/refresh.rs @@ -0,0 +1,3247 @@ +use super::{ + begin_hls_origin_account_io, finish_hls_origin_account_io, is_hls_provisioning_gap_segment, + is_hls_provisioning_segment, + manifest_fetch::{ + commit_error_to_fetch_error, evaluate_manifest_origin_quality_with_mode, fetch_hls_origin_manifest_request, + fetched_effective_manifest_host, log_hls_manifest_initial_selected, manifest_host_switch_failure_threshold, + manifest_origin_quality_from_candidate, next_committed_origin_highwater, + retry_hls_origin_manifest_recovery_chain, score_hls_manifest_candidate_for_selection_log, + FetchedOriginManifest, HlsManifestAcceptanceRejectReason, HlsManifestCommitAcceptanceMode, + HlsManifestCommitError, HlsManifestOriginQuality, HlsManifestOriginRelation, HlsManifestRejectLogReason, + HlsManifestSequenceRelation, HlsOriginManifestFetchContext, HlsOriginManifestFetchRequest, LiveHlsOriginEntry, + OriginManifestFetchError, RetryPolicy, + }, + safe_hls_access_lease_id, safe_origin_log_value, safe_proxy_session_id, safe_session_key, + hls_origin_headers_with_provider_session, sanitized_hls_origin_headers, + HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsFreshManifestRequiredReason, HlsManifestRenderer, + HlsManifestTemporaryFailureKind, HlsManifestTemporaryFailureTransition, HlsMapWorkerPool, HlsOriginIoContext, + HlsOriginWorkClass, HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionHandle, + HlsSessionMode, MapFetchContext, RenderedManifestStoreOutcome, RenderedManifestStoreRejectReason, + SegmentFetchContext, TransientPassthroughReason, +}; +use crate::{ + model::{AppConfig, HlsManifestRecoveryBurstConfig, ReverseProxyDisabledHeaderConfig, StripConfig}, + processing::parser::hls::{ + initial_strip::initial_hls_strip_segments_for_durations, + origin_manifest::{ + parse_manifest_timing, parse_manifest_validity, parse_origin_manifest_timeline, + parse_origin_media_manifest, OriginManifestParseOutcome, OriginManifestTransientReason, + ParsedOriginManifest, ParsedOriginManifestTimeline, + }, + transient_manifest::{ + apply_transient_discontinuity_sequence, materialize_transient_provisioning_handoff_view, + transient_discontinuity_sequence, transient_visible_discontinuity_count, TransientManifestRewriter, + TransientRewriteOptions, + }, + }, +}; +use axum::http::HeaderMap; +use log::{debug, info, warn}; +use reqwest::Client; +use shared::model::HlsStripMode; +use shared::utils::sanitize_sensitive_info; +use std::sync::Arc; +use url::Url; + +const COLD_START_RETRY_AFTER_SECONDS: u64 = 2; +const FIRST_FAILURE_BACKOFF_MS: u64 = 0; +const SECOND_FAILURE_BACKOFF_MS: u64 = 500; +const LATER_FAILURE_BACKOFF_MS: u64 = 1_000; + +/// Debounce and singleflight state for one live HLS origin manifest. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct OriginRefreshState { + pub last_fetch_started_at_ms: Option, + pub last_fetch_finished_at_ms: Option, + pub next_fetch_allowed_at_ms: u64, + pub consecutive_failures: u32, + pub consecutive_empty_refreshes: u32, + pub last_success_at_ms: Option, + pub last_error_at_ms: Option, + pub in_flight: bool, +} + +impl OriginRefreshState { + pub fn is_due(&self, now_ms: u64) -> bool { now_ms >= self.next_fetch_allowed_at_ms && !self.in_flight } + + pub fn mark_started(&mut self, now_ms: u64) { + self.last_fetch_started_at_ms = Some(now_ms); + self.in_flight = true; + } + + fn mark_success_with_timing( + &mut self, + fetch_started_at_ms: u64, + fetch_finished_at_ms: u64, + timing: HlsManifestRefreshTiming, + ) -> u64 { + let refresh_interval_ms = match timing.progress { + HlsManifestProgress::Advanced | HlsManifestProgress::Rollover => { + self.consecutive_empty_refreshes = 0; + timing.base_interval_ms + } + HlsManifestProgress::Unchanged => { + self.consecutive_empty_refreshes = self.consecutive_empty_refreshes.saturating_add(1); + apply_empty_refresh_rampdown_ms(timing.base_interval_ms, self.consecutive_empty_refreshes) + } + }; + self.last_fetch_finished_at_ms = Some(fetch_finished_at_ms); + self.last_success_at_ms = Some(fetch_finished_at_ms); + self.last_error_at_ms = None; + self.consecutive_failures = 0; + self.in_flight = false; + self.next_fetch_allowed_at_ms = fetch_started_at_ms.saturating_add(refresh_interval_ms); + refresh_interval_ms + } + + pub fn mark_failure(&mut self, failed_at_ms: u64) { + self.last_fetch_finished_at_ms = Some(failed_at_ms); + self.last_error_at_ms = Some(failed_at_ms); + let next_retry_delay_ms = self.next_failure_backoff_ms(); + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.in_flight = false; + self.next_fetch_allowed_at_ms = failed_at_ms.saturating_add(next_retry_delay_ms); + } + + fn next_failure_backoff_ms(&self) -> u64 { + match self.consecutive_failures { + 0 => FIRST_FAILURE_BACKOFF_MS, + 1 => SECOND_FAILURE_BACKOFF_MS, + _ => LATER_FAILURE_BACKOFF_MS, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum HlsManifestAcceptanceDecision { + Accept { quality: HlsManifestOriginQuality }, + RetryCurrentTarget { quality: HlsManifestOriginQuality }, + AcceptHostSwitch { quality: HlsManifestOriginQuality }, + Reject { reason: HlsManifestAcceptanceRejectReason, quality: HlsManifestOriginQuality }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum HlsManifestProgress { + Advanced, + Rollover, + Unchanged, +} + +impl HlsManifestProgress { + fn as_log_value(self) -> &'static str { + match self { + Self::Advanced => "advanced", + Self::Rollover => "rollover", + Self::Unchanged => "unchanged", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum HlsManifestTimingSource { + LastSegmentDuration, + TargetDuration, + Fallback, +} + +impl HlsManifestTimingSource { + fn as_log_value(self) -> &'static str { + match self { + Self::LastSegmentDuration => "last_segment_duration", + Self::TargetDuration => "target_duration", + Self::Fallback => "fallback", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +struct HlsManifestRefreshTiming { + last_segment_duration_ms: Option, + target_duration_ms: Option, + base_interval_ms: u64, + source: HlsManifestTimingSource, + progress: HlsManifestProgress, +} + +#[derive(Clone)] +pub struct OriginRefreshRequest { + pub app_config: Arc, + pub session: HlsSessionHandle, + pub origin_entry: LiveHlsOriginEntry, + pub headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub disabled_headers: Option, + pub client: Client, + pub no_redirect_client: Client, + pub use_manual_redirects: bool, + pub segment_cache: Arc, + pub hls_proxy: Arc, + pub segment_repair: Arc, + pub segment_worker_pool: Arc, + pub map_worker_pool: Arc, + pub origin_manifest_timeout_ms: u64, + pub manifest_recovery_burst: HlsManifestRecoveryBurstConfig, + pub strip: StripConfig, + pub retry_policy: RetryPolicy, + pub reverse_proxy_rewrite_secret: Vec, + pub transient_resource_ttl_ms: u64, + pub manifest_commit_requirement: HlsManifestCommitRequirement, + pub access_lease_id: Option, + pub now_ms: u64, + pub origin_io: Option, +} + +/// Controls whether a canonical HLS refresh may rely on an existing committed manifest. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsManifestCommitRequirement { + CommittedManifestAllowed, + FreshCommitRequired { reason: HlsFreshManifestRequiredReason }, +} + +impl HlsManifestCommitRequirement { + const fn fresh_reason(self) -> Option { + match self { + Self::CommittedManifestAllowed => None, + Self::FreshCommitRequired { reason } => Some(reason), + } + } + + const fn acceptance_mode(self) -> HlsManifestCommitAcceptanceMode { + match self { + Self::CommittedManifestAllowed => HlsManifestCommitAcceptanceMode::StrictPinnedHost, + Self::FreshCommitRequired { .. } => HlsManifestCommitAcceptanceMode::FreshBaseline, + } + } +} + +pub async fn maybe_trigger_origin_refresh(mut request: OriginRefreshRequest) -> bool { + let fetch_started_at_ms = request.now_ms; + if !mark_origin_refresh_started(&mut request, fetch_started_at_ms).await { + release_preacquired_origin_provider_handle(&request).await; + return false; + } + + tokio::spawn(async move { + refresh_and_commit(request, fetch_started_at_ms).await; + }); + true +} + +pub async fn trigger_origin_refresh_sync(mut request: OriginRefreshRequest) -> bool { + let fetch_started_at_ms = request.now_ms; + if !mark_origin_refresh_started(&mut request, fetch_started_at_ms).await { + release_preacquired_origin_provider_handle(&request).await; + return false; + } + + refresh_and_commit(request, fetch_started_at_ms).await; + true +} + +async fn mark_origin_refresh_started(request: &mut OriginRefreshRequest, fetch_started_at_ms: u64) -> bool { + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + { + let mut session = request.session.write().await; + if session.is_gc_marked_for_removal() { + return false; + } + if session.origin_refresh.in_flight { + metrics.record_refresh_skipped(); + let last_fetch_started_at_ms = session + .origin_refresh + .last_fetch_started_at_ms + .map_or_else(|| "".to_string(), |started_at_ms| started_at_ms.to_string()); + let in_flight_for_ms = session.origin_refresh.last_fetch_started_at_ms.map_or_else( + || "".to_string(), + |started_at_ms| fetch_started_at_ms.saturating_sub(started_at_ms).to_string(), + ); + debug!( + "HLS origin manifest refresh skipped: session={} proxy_session_id={} reason=in_flight last_fetch_started_at_ms={} in_flight_for_ms={} now_ms={fetch_started_at_ms}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + last_fetch_started_at_ms, + in_flight_for_ms + ); + return false; + } + if fetch_started_at_ms < session.origin_refresh.next_fetch_allowed_at_ms + && request.manifest_commit_requirement.fresh_reason().is_none() + { + metrics.record_refresh_skipped(); + let wait_ms = session.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(fetch_started_at_ms); + debug!( + "HLS origin manifest refresh skipped: session={} proxy_session_id={} reason=debounce next_fetch_allowed_at_ms={} now_ms={fetch_started_at_ms} wait_ms={wait_ms}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + session.origin_refresh.next_fetch_allowed_at_ms + ); + return false; + } + if let Some(origin_io) = request.origin_io.as_mut() { + origin_io.started_generation = Some(session.start_origin_work()); + } + session.origin_refresh.mark_started(fetch_started_at_ms); + metrics.record_refresh_started(); + info!( + "HLS origin manifest refresh started: session={} proxy_session_id={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id) + ); + } + true +} + +#[allow(clippy::too_many_lines)] +async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ms: u64) { + request.headers = sanitized_hls_origin_headers(&request.headers, request.disabled_headers.as_ref()); + let provider_lease = if let Some(origin_io) = request.origin_io.as_ref() { + let binding = request.session.read().await.origin_account_binding.clone(); + if let Some(binding) = binding { + match begin_hls_origin_account_io(origin_io, &request.session, &binding).await { + Ok(guard) => { + debug!( + "HLS provider session lease joined for manifest refresh: provider={}", + sanitize_sensitive_info(binding.account_name.as_ref()) + ); + Some((origin_io.clone(), guard)) + } + Err(kind) => { + touch_refresh_origin_account_binding(&request, false).await; + let _ = finish_refresh_origin_work(&request, current_time_millis()).await; + finish_refresh_failure(&request, OriginManifestFetchError::ProviderUnavailable(kind)).await; + return; + } + } + } else { + None + } + } else { + None + }; + let result = fetch_and_commit_manifest_with_policy(&mut request).await; + let fetch_finished_at_ms = current_time_millis(); + let origin_work_state = finish_refresh_origin_work(&request, fetch_finished_at_ms).await; + if let Some((origin_io, guard)) = provider_lease { + let binding = guard.binding().clone(); + finish_hls_origin_account_io( + &origin_io, + &request.session, + guard, + origin_work_state.generation_valid && origin_work_state.refresh_reservation, + ) + .await; + debug!( + "HLS provider session lease released after manifest refresh: provider={}", + sanitize_sensitive_info(binding.account_name.as_ref()) + ); + touch_refresh_origin_account_binding( + &request, + origin_work_state.generation_valid && origin_work_state.refresh_reservation, + ) + .await; + } + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + let ( + should_wake_segment_scheduler, + should_wake_map_scheduler, + fresh_manifest_failure_reason, + temporary_manifest_failure_reason, + pending_manifest_follow_up, + ) = { + let mut session = request.session.write().await; + match result { + Ok(CommittedOriginManifest { fetched, refresh_timing, wake_segment_scheduler, wake_map_scheduler }) => { + let pending_manifest_follow_up = Some((session.proxy_session_id.clone(), session.target_duration)); + let applied_refresh_interval_ms = session.origin_refresh.mark_success_with_timing( + fetch_started_at_ms, + fetch_finished_at_ms, + refresh_timing, + ); + if let Some(reset_failures) = session.record_successful_manifest_fetch() { + debug!( + "HLS manifest temporary failure counter reset: session={} previous_failures={reset_failures}", + safe_session_key(&session.key) + ); + } + log_manifest_refresh_timing(&session, refresh_timing, applied_refresh_interval_ms); + metrics.record_refresh_completed(); + for _ in 1..fetched.attempts { + metrics.record_refresh_retried(); + } + info!( + "HLS origin manifest refresh completed: session={} proxy_session_id={} final_url={} redirect_host={} status={} attempts={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + safe_origin_log_value(&fetched.final_manifest_url), + fetched.redirect_host.as_deref().unwrap_or(""), + fetched.status.as_u16(), + fetched.attempts + ); + if origin_work_state.generation_valid { + (wake_segment_scheduler, wake_map_scheduler, None, None, pending_manifest_follow_up) + } else { + session.invalidate_queued_origin_work(); + (false, false, None, None, pending_manifest_follow_up) + } + } + Err(err) => { + session.origin_refresh.mark_failure(fetch_finished_at_ms); + metrics.record_refresh_failed(); + let temporary_manifest_failure_reason = record_temporary_manifest_fetch_failure_if_needed( + &mut session, + &request.strip, + &err, + fetch_finished_at_ms, + ); + if manifest_hard_fetch_error(&err) { + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + debug!( + "HLS manifest marked fresh-commit required after hard fetch failure: session={}", + safe_session_key(&session.key) + ); + } + warn!( + "HLS origin manifest refresh completed: session={} proxy_session_id={} result=failed error={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + safe_origin_log_value(format!("{err:?}")) + ); + ( + false, + false, + request.manifest_commit_requirement.fresh_reason(), + temporary_manifest_failure_reason, + None, + ) + } + } + }; + if let Some((proxy_session_id, target_duration)) = pending_manifest_follow_up { + let shortened = request + .hls_proxy + .mark_pending_manifest_follow_up_for_session(&proxy_session_id, fetch_finished_at_ms, target_duration) + .await; + if shortened > 0 { + debug!( + "HLS pending manifest leases shortened after manifest commit: session={} leases={shortened}", + safe_proxy_session_id(&proxy_session_id) + ); + } + } + if let Some(reason) = fresh_manifest_failure_reason { + mark_fresh_manifest_commit_failed_access_leases(&request, fetch_finished_at_ms, reason).await; + } + if let Some((failures, threshold)) = temporary_manifest_failure_reason { + mark_manifest_temporary_failure_access_leases(&request, fetch_finished_at_ms, failures, threshold).await; + } + + let origin_provider_session_headers = request.session.read().await.origin_provider_session_headers.clone(); + + if should_wake_map_scheduler { + request + .map_worker_pool + .wake_scheduler( + MapFetchContext { + session: Arc::clone(&request.session), + segment_cache: Arc::clone(&request.segment_cache), + headers: request.headers.clone(), + origin_provider_session_headers: origin_provider_session_headers.clone(), + client: request.client.clone(), + no_redirect_client: request.no_redirect_client.clone(), + use_manual_redirects: request.use_manual_redirects, + origin_io: request + .origin_io + .clone() + .map(|origin_io| origin_io.with_grace(HlsOriginWorkClass::Background.allows_grace())), + }, + fetch_finished_at_ms, + ) + .await; + } + + if should_wake_segment_scheduler { + request + .segment_worker_pool + .wake_scheduler( + SegmentFetchContext { + session: request.session, + segment_cache: request.segment_cache, + segment_repair: request.segment_repair, + repair_access_lease_id: None, + headers: request.headers, + origin_provider_session_headers, + client: request.client, + no_redirect_client: request.no_redirect_client, + use_manual_redirects: request.use_manual_redirects, + origin_io: request + .origin_io + .map(|origin_io| origin_io.with_grace(HlsOriginWorkClass::Background.allows_grace())), + }, + fetch_finished_at_ms, + ) + .await; + } +} + +async fn mark_fresh_manifest_commit_failed_access_leases( + request: &OriginRefreshRequest, + failed_at_ms: u64, + reason: HlsFreshManifestRequiredReason, +) { + let Some(origin_io) = request.origin_io.as_ref() else { + return; + }; + let unavailable_reason = HlsAccessLeaseChannelUnavailableReason::ManifestCommitFailed { reason }; + if let Some(access_lease_id) = request.access_lease_id.as_ref() { + let marked = origin_io + .app_state + .hls_proxy + .mark_access_lease_channel_unavailable(access_lease_id, failed_at_ms, unavailable_reason) + .await; + if marked { + let proxy_session_id = request.session.read().await.proxy_session_id.clone(); + debug!( + "HLS access lease marked channel unavailable after fresh manifest commit failed: session={} lease={} reason={reason:?}", + safe_proxy_session_id(&proxy_session_id), + safe_hls_access_lease_id(access_lease_id) + ); + } + return; + } + + let proxy_session_id = request.session.read().await.proxy_session_id.clone(); + let marked = origin_io + .app_state + .hls_proxy + .mark_access_leases_channel_unavailable_for_session( + &proxy_session_id, + failed_at_ms, + unavailable_reason, + ) + .await; + if marked > 0 { + debug!( + "HLS access leases marked channel unavailable after fresh manifest commit failed: session={} marked={marked} reason={reason:?}", + safe_proxy_session_id(&proxy_session_id) + ); + } +} + +fn record_temporary_manifest_fetch_failure_if_needed( + session: &mut super::HlsSession, + strip: &StripConfig, + err: &OriginManifestFetchError, + failed_at_ms: u64, +) -> Option<(u32, u32)> { + let kind = manifest_temporary_failure_kind(err)?; + let threshold = manifest_host_switch_failure_threshold(session, strip); + match session.record_temporary_manifest_fetch_failure(failed_at_ms, kind, threshold) { + HlsManifestTemporaryFailureTransition::StillRetryable { failures, threshold } => { + debug!( + "HLS manifest temporary failure counted: session={} failures={} threshold={}", + safe_session_key(&session.key), + failures, + threshold + ); + None + } + HlsManifestTemporaryFailureTransition::BecameChannelUnavailable { failures, threshold } => { + debug!( + "HLS manifest temporary failure threshold reached: session={} failures={} threshold={}", + safe_session_key(&session.key), + failures, + threshold + ); + Some((failures, threshold)) + } + } +} + +fn manifest_temporary_failure_kind(err: &OriginManifestFetchError) -> Option { + match err { + OriginManifestFetchError::Timeout => Some(HlsManifestTemporaryFailureKind::Timeout), + OriginManifestFetchError::RetryableStatus(status, _) => { + Some(HlsManifestTemporaryFailureKind::RetryableStatus { status: *status }) + } + OriginManifestFetchError::Request(message) if request_error_indicates_timeout(message) => { + Some(HlsManifestTemporaryFailureKind::Timeout) + } + OriginManifestFetchError::ProviderUnavailable(kind) if kind.is_retryable_resource_failure() => { + Some(HlsManifestTemporaryFailureKind::ProviderAcquire { kind: *kind }) + } + OriginManifestFetchError::PermanentStatus(_) + | OriginManifestFetchError::RetryExhausted + | OriginManifestFetchError::NonRetryableStatus(_) + | OriginManifestFetchError::Request(_) + | OriginManifestFetchError::Redirect(_) + | OriginManifestFetchError::ProviderUnavailable(_) => None, + } +} + +fn manifest_hard_fetch_error(err: &OriginManifestFetchError) -> bool { + match err { + OriginManifestFetchError::PermanentStatus(_) | OriginManifestFetchError::NonRetryableStatus(_) => true, + OriginManifestFetchError::ProviderUnavailable(kind) => !kind.is_retryable_resource_failure(), + OriginManifestFetchError::RetryableStatus(_, _) + | OriginManifestFetchError::RetryExhausted + | OriginManifestFetchError::Request(_) + | OriginManifestFetchError::Redirect(_) + | OriginManifestFetchError::Timeout => false, + } +} + +fn request_error_indicates_timeout(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + lower.contains("timeout") || lower.contains("timed out") +} + +async fn mark_manifest_temporary_failure_access_leases( + request: &OriginRefreshRequest, + failed_at_ms: u64, + failures: u32, + threshold: u32, +) { + let Some(origin_io) = request.origin_io.as_ref() else { + return; + }; + let proxy_session_id = request.session.read().await.proxy_session_id.clone(); + let marked = origin_io + .app_state + .hls_proxy + .mark_access_leases_channel_unavailable_for_session( + &proxy_session_id, + failed_at_ms, + HlsAccessLeaseChannelUnavailableReason::ManifestTemporaryFailureThreshold { failures, threshold }, + ) + .await; + if marked > 0 { + debug!( + "HLS access leases marked channel unavailable after temporary manifest failures: session={} marked={marked} failures={} threshold={}", + safe_proxy_session_id(&proxy_session_id), + failures, + threshold + ); + } +} + +#[derive(Clone, Copy)] +struct OriginWorkFinishState { + generation_valid: bool, + refresh_reservation: bool, +} + +async fn finish_refresh_origin_work(request: &OriginRefreshRequest, now_ms: u64) -> OriginWorkFinishState { + let Some(started_generation) = request.origin_io.as_ref().and_then(|origin_io| origin_io.started_generation) else { + return OriginWorkFinishState { generation_valid: true, refresh_reservation: false }; + }; + let mut session = request.session.write().await; + let generation_valid = session.finish_origin_work(started_generation); + let refresh_reservation = session.should_refresh_origin_reservation(now_ms); + OriginWorkFinishState { generation_valid, refresh_reservation } +} + +async fn finish_refresh_failure(request: &OriginRefreshRequest, err: OriginManifestFetchError) { + release_preacquired_origin_provider_handle(request).await; + let fetch_finished_at_ms = current_time_millis(); + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + let (fresh_manifest_failure_reason, temporary_manifest_failure_reason) = { + let mut session = request.session.write().await; + session.origin_refresh.mark_failure(fetch_finished_at_ms); + metrics.record_refresh_failed(); + let temporary_manifest_failure_reason = + record_temporary_manifest_fetch_failure_if_needed(&mut session, &request.strip, &err, fetch_finished_at_ms); + if manifest_hard_fetch_error(&err) { + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + debug!( + "HLS manifest marked fresh-commit required after hard fetch failure: session={}", + safe_session_key(&session.key) + ); + } + warn!( + "HLS origin manifest refresh completed: session={} proxy_session_id={} result=failed error={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + safe_origin_log_value(format!("{err:?}")) + ); + (request.manifest_commit_requirement.fresh_reason(), temporary_manifest_failure_reason) + }; + if let Some(reason) = fresh_manifest_failure_reason { + mark_fresh_manifest_commit_failed_access_leases(request, fetch_finished_at_ms, reason).await; + } + if let Some((failures, threshold)) = temporary_manifest_failure_reason { + mark_manifest_temporary_failure_access_leases(request, fetch_finished_at_ms, failures, threshold).await; + } +} + +async fn release_preacquired_origin_provider_handle(request: &OriginRefreshRequest) { + let Some(origin_io) = request.origin_io.as_ref() else { + return; + }; + let Some(handle) = origin_io.take_preacquired_provider_handle().await else { + return; + }; + let binding = request.session.read().await.origin_account_binding.clone(); + if let Some(binding) = binding { + origin_io.app_state.connection_manager.release_provider_handle(Some(handle)).await; + debug!( + "HLS provider handle released after manifest refresh: provider={} reason=refresh-not-started", + sanitize_sensitive_info(binding.account_name.as_ref()) + ); + } else { + origin_io.app_state.connection_manager.release_provider_handle(Some(handle)).await; + } +} + +async fn touch_refresh_origin_account_binding(request: &OriginRefreshRequest, reservation_refreshed: bool) { + let mut session = request.session.write().await; + if let Some(binding) = session.origin_account_binding.as_mut() { + let now_ms = current_time_millis(); + binding.last_origin_io_at_ms = Some(now_ms); + if reservation_refreshed { + binding.last_reservation_refresh_at_ms = Some(now_ms); + } + } +} + +struct CommittedOriginManifest { + fetched: FetchedOriginManifest, + refresh_timing: HlsManifestRefreshTiming, + wake_segment_scheduler: bool, + wake_map_scheduler: bool, +} + +fn manifest_fetch_context(request: &OriginRefreshRequest) -> HlsOriginManifestFetchContext { + HlsOriginManifestFetchContext { + app_config: Arc::clone(&request.app_config), + session: Arc::clone(&request.session), + origin_entry: request.origin_entry.clone(), + headers: hls_origin_headers_with_provider_session(&request.headers, &request.origin_provider_session_headers), + client: request.client.clone(), + no_redirect_client: request.no_redirect_client.clone(), + use_manual_redirects: request.use_manual_redirects, + origin_manifest_timeout_ms: request.origin_manifest_timeout_ms, + manifest_recovery_burst: request.manifest_recovery_burst.clone(), + retry_policy: request.retry_policy.clone(), + } +} + +#[allow(clippy::too_many_lines)] +async fn fetch_and_commit_manifest_with_policy( + request: &mut OriginRefreshRequest, +) -> Result { + let fetch_context = manifest_fetch_context(request); + let fetched = + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&fetch_context)) + .await? + .with_attempts(1); + let acceptance_mode = request.manifest_commit_requirement.acceptance_mode(); + let selected_report = + score_hls_manifest_candidate_for_selection_log(&fetch_context, &fetched, acceptance_mode).await; + let commit_result = { + let mut session = request.session.write().await; + commit_fetched_manifest(&mut session, &fetched, request, current_time_millis()) + }; + + match commit_result { + Ok((refresh_timing, wake_segment_scheduler, wake_map_scheduler)) => { + if let Some(report) = selected_report.as_ref() { + log_hls_manifest_initial_selected(&fetch_context, report).await; + } + Ok(CommittedOriginManifest { fetched, refresh_timing, wake_segment_scheduler, wake_map_scheduler }) + } + Err(HlsManifestCommitError::RetryCurrentTarget) => { + let target_url = Url::parse(&fetched.resolved_request_url) + .map_err(|err| OriginManifestFetchError::Request(err.to_string()))?; + let last_error = match retry_hls_origin_manifest_recovery_chain( + &fetch_context, + target_url, + fetched.provider_url_index, + None, + |fetched, acceptance_mode| commit_manifest_recovery_candidate(request, fetched, acceptance_mode), + ) + .await + { + Ok(committed) => return Ok(committed), + Err(err) => err, + }; + + let decision = { + let mut session = request.session.write().await; + record_pinned_host_recovery_chain_failed(&mut session, &request.strip, current_time_millis()) + }; + match decision { + HlsManifestAcceptanceDecision::AcceptHostSwitch { .. } => { + let candidate_commit_result = { + let mut session = request.session.write().await; + commit_fetched_manifest_with_acceptance_mode( + &mut session, + &fetched, + request, + current_time_millis(), + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ) + }; + match candidate_commit_result { + Ok((refresh_timing, wake_segment_scheduler, wake_map_scheduler)) => { + if let Some(report) = selected_report.as_ref() { + log_hls_manifest_initial_selected(&fetch_context, report).await; + } + Ok(CommittedOriginManifest { + fetched, + refresh_timing, + wake_segment_scheduler, + wake_map_scheduler, + }) + } + Err(err) => Err(commit_error_to_fetch_error(&err)), + } + } + HlsManifestAcceptanceDecision::Reject { reason, .. } => { + debug!( + "HLS origin manifest host switch held: origin_entry={} reason={reason:?}", + safe_origin_log_value(request.origin_entry.url().as_str()) + ); + Err(last_error) + } + HlsManifestAcceptanceDecision::Accept { .. } + | HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => Err(last_error), + } + } + Err(HlsManifestCommitError::TimelineRejected { reason }) => { + let target_url = + Url::parse(&fetched.resolved_request_url).map_err(|_| OriginManifestFetchError::RetryExhausted)?; + retry_hls_origin_manifest_recovery_chain( + &fetch_context, + target_url, + fetched.provider_url_index, + Some(reason), + |fetched, acceptance_mode| commit_manifest_recovery_candidate(request, fetched, acceptance_mode), + ) + .await + } + } +} + +async fn commit_manifest_recovery_candidate( + request: &OriginRefreshRequest, + fetched: FetchedOriginManifest, + acceptance_mode: HlsManifestCommitAcceptanceMode, +) -> Result { + let commit_result = { + let mut session = request.session.write().await; + commit_fetched_manifest_with_acceptance_mode( + &mut session, + &fetched, + request, + current_time_millis(), + acceptance_mode, + ) + }; + match commit_result { + Ok((refresh_timing, wake_segment_scheduler, wake_map_scheduler)) => { + Ok(CommittedOriginManifest { fetched, refresh_timing, wake_segment_scheduler, wake_map_scheduler }) + } + Err(err) => Err(err), + } +} + +fn commit_fetched_manifest( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + request: &OriginRefreshRequest, + fetch_finished_at_ms: u64, +) -> Result<(HlsManifestRefreshTiming, bool, bool), HlsManifestCommitError> { + commit_fetched_manifest_with_acceptance_mode( + session, + fetched, + request, + fetch_finished_at_ms, + request.manifest_commit_requirement.acceptance_mode(), + ) +} + +fn commit_fetched_manifest_with_acceptance_mode( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + request: &OriginRefreshRequest, + fetch_finished_at_ms: u64, + acceptance_mode: HlsManifestCommitAcceptanceMode, +) -> Result<(HlsManifestRefreshTiming, bool, bool), HlsManifestCommitError> { + let existing_transient_reason = match &session.mode { + HlsSessionMode::TransientPassthrough { reason } => Some(reason.clone()), + HlsSessionMode::NormalCacheTimeline => None, + }; + + match (existing_transient_reason, parse_origin_media_manifest(&fetched.body, &fetched.final_manifest_url)) { + (None, OriginManifestParseOutcome::Normal(manifest)) => { + let quality = evaluate_manifest_acceptance_for_commit( + session, + fetched, + ParsedOriginManifestTimeline { + origin_manifest_sequence: manifest.origin_manifest_sequence, + origin_manifest_segment_cnt: manifest.origin_manifest_segment_cnt, + }, + request, + fetch_finished_at_ms, + acceptance_mode, + )?; + mark_manifest_handoff_discontinuity_if_needed(session, &quality); + let result = commit_normal_manifest( + session, + &manifest, + fetched.redirect_host.as_deref(), + request, + &fetched.resolved_request_url, + fetch_finished_at_ms, + quality.sequence_relation, + ); + result.map(|refresh_timing| { + update_origin_provider_session_headers(session, fetched); + mark_manifest_acceptance_success(session, fetched, &quality); + (refresh_timing, true, true) + }) + } + (Some(reason), _) => { + let timeline = parse_transient_manifest_timeline_for_commit(session, &fetched.body)?; + let quality = evaluate_manifest_acceptance_for_commit( + session, + fetched, + timeline, + request, + fetch_finished_at_ms, + acceptance_mode, + )?; + mark_manifest_handoff_discontinuity_if_needed(session, &quality); + let refresh_timing = commit_transient_manifest( + session, + &fetched.body, + &fetched.final_manifest_url, + &fetched.resolved_request_url, + fetched.redirect_host.as_deref(), + &request.headers, + reason, + &request.reverse_proxy_rewrite_secret, + request.transient_resource_ttl_ms, + fetch_finished_at_ms, + timeline, + &quality, + &request.strip, + ); + update_origin_provider_session_headers(session, fetched); + mark_manifest_acceptance_success(session, fetched, &quality); + Ok((refresh_timing, false, false)) + } + (None, OriginManifestParseOutcome::TransientPassthrough { reason }) => { + let timeline = parse_transient_manifest_timeline_for_commit(session, &fetched.body)?; + let quality = evaluate_manifest_acceptance_for_commit( + session, + fetched, + timeline, + request, + fetch_finished_at_ms, + acceptance_mode, + )?; + request.segment_worker_pool.metrics().record_transient_switch(); + mark_manifest_handoff_discontinuity_if_needed(session, &quality); + let refresh_timing = commit_transient_manifest( + session, + &fetched.body, + &fetched.final_manifest_url, + &fetched.resolved_request_url, + fetched.redirect_host.as_deref(), + &request.headers, + map_transient_reason(reason), + &request.reverse_proxy_rewrite_secret, + request.transient_resource_ttl_ms, + fetch_finished_at_ms, + timeline, + &quality, + &request.strip, + ); + update_origin_provider_session_headers(session, fetched); + mark_manifest_acceptance_success(session, fetched, &quality); + Ok((refresh_timing, false, false)) + } + } +} + +fn update_origin_provider_session_headers(session: &mut super::HlsSession, fetched: &FetchedOriginManifest) { + if !fetched.provider_session_headers.is_empty() { + session.origin_provider_session_headers = fetched.provider_session_headers.clone(); + } +} + +fn parse_transient_manifest_timeline_for_commit( + session: &super::HlsSession, + body: &str, +) -> Result { + parse_origin_manifest_timeline(body).map_err(|reason| { + warn!( + "HLS origin manifest rejected: session={} proxy_session_id={} reason=malformed-transient-timeline error={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + safe_origin_log_value(format!("{reason:?}")) + ); + HlsManifestCommitError::TimelineRejected { reason: HlsManifestRejectLogReason::MalformedTransientTimeline } + }) +} + +fn evaluate_manifest_acceptance_for_commit( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + timeline: ParsedOriginManifestTimeline, + request: &OriginRefreshRequest, + now_ms: u64, + mode: HlsManifestCommitAcceptanceMode, +) -> Result { + let fetch_context = manifest_fetch_context(request); + match evaluate_manifest_acceptance(session, fetched, timeline, &fetch_context, now_ms, mode) { + HlsManifestAcceptanceDecision::Accept { quality } + | HlsManifestAcceptanceDecision::AcceptHostSwitch { quality } => Ok(quality), + HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => Err(HlsManifestCommitError::RetryCurrentTarget), + HlsManifestAcceptanceDecision::Reject { reason, .. } => { + let log_reason = HlsManifestRejectLogReason::from(reason.clone()); + warn!( + "HLS origin manifest rejected: session={} proxy_session_id={} reason={} media_sequence={} segments={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + log_reason.status_label(), + timeline.origin_manifest_sequence, + timeline.origin_manifest_segment_cnt + ); + Err(HlsManifestCommitError::TimelineRejected { reason: log_reason }) + } + } +} + +fn evaluate_manifest_acceptance( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + timeline: ParsedOriginManifestTimeline, + fetch_context: &HlsOriginManifestFetchContext, + now_ms: u64, + mode: HlsManifestCommitAcceptanceMode, +) -> HlsManifestAcceptanceDecision { + let quality = evaluate_manifest_origin_quality_with_mode(session, fetched, timeline, fetch_context, now_ms, mode); + + match quality.host_relation { + HlsManifestOriginRelation::Initial + | HlsManifestOriginRelation::SameRedirectHost + | HlsManifestOriginRelation::UnknownHost => { + if let Some(reason) = quality.reject_reason.clone() { + return HlsManifestAcceptanceDecision::Reject { reason, quality }; + } + HlsManifestAcceptanceDecision::Accept { quality } + } + HlsManifestOriginRelation::OtherRedirectHost => { + let Some(effective_host) = quality.effective_host.clone() else { + return HlsManifestAcceptanceDecision::Accept { quality }; + }; + + upsert_host_switch_candidate(session, fetched, &quality, effective_host.clone(), now_ms); + if matches!(mode, HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate) + && session + .manifest_acceptance + .host_switch_candidate + .as_ref() + .is_some_and(|candidate| candidate.host == effective_host) + { + if let Some(reason) = quality.reject_reason.clone() { + return HlsManifestAcceptanceDecision::Reject { reason, quality }; + } + return HlsManifestAcceptanceDecision::AcceptHostSwitch { quality }; + } + + HlsManifestAcceptanceDecision::RetryCurrentTarget { quality } + } + } +} + +fn mark_manifest_handoff_discontinuity_if_needed( + session: &mut super::HlsSession, + quality: &HlsManifestOriginQuality, +) { + if !quality.requires_handoff_discontinuity { + return; + } + if matches!(quality.host_relation, HlsManifestOriginRelation::OtherRedirectHost) { + session.mark_pending_origin_epoch_handoff_discontinuity(0); + } else if session.pending_handoff_discontinuity_sequence.is_none() { + session.mark_pending_handoff_discontinuity(0); + } +} + +fn upsert_host_switch_candidate( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + quality: &HlsManifestOriginQuality, + host: String, + now_ms: u64, +) { + match session.manifest_acceptance.host_switch_candidate.as_mut() { + Some(candidate) if candidate.host == host => { + candidate.target_url.clone_from(&fetched.resolved_request_url); + candidate.last_seen_at_ms = now_ms; + candidate.seen_count = candidate.seen_count.saturating_add(1); + candidate.highwater = quality.origin_highwater; + candidate.quality_score = quality.score.rank(); + } + _ => { + session.manifest_acceptance.host_switch_candidate = Some(super::HlsManifestHostSwitchCandidate { + host, + target_url: fetched.resolved_request_url.clone(), + first_seen_at_ms: now_ms, + last_seen_at_ms: now_ms, + seen_count: 1, + highwater: quality.origin_highwater, + quality_score: quality.score.rank(), + }); + } + } +} + +fn record_pinned_host_recovery_chain_failed( + session: &mut super::HlsSession, + strip: &StripConfig, + now_ms: u64, +) -> HlsManifestAcceptanceDecision { + let Some(candidate) = session.manifest_acceptance.host_switch_candidate.as_mut() else { + let quality = manifest_origin_quality_from_candidate(None); + return HlsManifestAcceptanceDecision::Reject { + reason: HlsManifestAcceptanceRejectReason::MissingPinnedTarget, + quality, + }; + }; + candidate.last_seen_at_ms = now_ms; + let failures = session.manifest_acceptance.same_host_retry_chain_failures.saturating_add(1); + session.manifest_acceptance.same_host_retry_chain_failures = failures; + let threshold = manifest_host_switch_failure_threshold(session, strip); + let quality = manifest_origin_quality_from_candidate(session.manifest_acceptance.host_switch_candidate.as_ref()); + if failures >= threshold { + HlsManifestAcceptanceDecision::AcceptHostSwitch { quality } + } else { + HlsManifestAcceptanceDecision::Reject { + reason: HlsManifestAcceptanceRejectReason::HostSwitchPending { failures, threshold }, + quality, + } + } +} + +fn mark_manifest_acceptance_success( + session: &mut super::HlsSession, + fetched: &FetchedOriginManifest, + quality: &HlsManifestOriginQuality, +) { + if let Some(effective_host) = fetched_effective_manifest_host(fetched) { + session.last_effective_manifest_host = Some(effective_host); + } + if quality.should_reset_stall_counter { + session.manifest_acceptance.same_host_retry_chain_failures = 0; + session.manifest_acceptance.host_switch_candidate = None; + } else if quality.should_increment_stall_counter { + session.manifest_acceptance.same_host_retry_chain_failures = + session.manifest_acceptance.same_host_retry_chain_failures.saturating_add(1); + } + if matches!(quality.host_relation, HlsManifestOriginRelation::OtherRedirectHost) { + session.manifest_acceptance.host_switch_candidate = None; + } +} + +#[allow(clippy::too_many_arguments)] +fn commit_transient_manifest( + session: &mut super::HlsSession, + body: &str, + final_manifest_url: &str, + _resolved_request_url: &str, + _redirect_host: Option<&str>, + request_headers: &HeaderMap, + reason: TransientPassthroughReason, + reverse_proxy_rewrite_secret: &[u8], + transient_resource_ttl_ms: u64, + rendered_at_ms: u64, + timeline: ParsedOriginManifestTimeline, + quality: &HlsManifestOriginQuality, + strip: &StripConfig, +) -> HlsManifestRefreshTiming { + let was_normal = matches!(session.mode, HlsSessionMode::NormalCacheTimeline); + let reason_log_fields = transient_reason_log_fields(&reason); + session.mode = HlsSessionMode::TransientPassthrough { reason }; + if was_normal { + info!( + "HLS session switched to transient passthrough: session={} proxy_session_id={} {}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + reason_log_fields + ); + } + session.origin_request_headers = request_headers.clone(); + session.transient.set_resource_ttl_ms(transient_resource_ttl_ms); + session.transient.prune_expired(rendered_at_ms); + + let previous_provisioning_manifest_body = session + .last_rendered_manifest + .as_ref() + .filter(|_| session.segments.values().any(is_hls_provisioning_segment)) + .map(|rendered| rendered.body.clone()); + let provisioning_segment_duration_ms = session + .segments + .values() + .find(|entry| is_hls_provisioning_segment(entry) || is_hls_provisioning_gap_segment(entry)) + .map_or(2_000, |entry| entry.duration_ms); + let handoff_discontinuity_sequence = session.take_pending_handoff_discontinuity_sequence(); + let mut rewritten = if handoff_discontinuity_sequence.is_some() { + TransientManifestRewriter::rewrite_with_options( + body, + final_manifest_url, + &session.proxy_session_id, + reverse_proxy_rewrite_secret, + rendered_at_ms, + transient_resource_ttl_ms, + TransientRewriteOptions { handoff_discontinuity_sequence }, + ) + } else { + TransientManifestRewriter::rewrite( + body, + final_manifest_url, + &session.proxy_session_id, + reverse_proxy_rewrite_secret, + rendered_at_ms, + transient_resource_ttl_ms, + ) + }; + if handoff_discontinuity_sequence.is_some() { + if let Some(handoff_body) = materialize_transient_provisioning_handoff_view( + &rewritten.body, + previous_provisioning_manifest_body.as_deref(), + strip, + provisioning_segment_duration_ms, + ) { + rewritten.body = handoff_body; + } + let current_discontinuity_sequence = transient_discontinuity_sequence(&rewritten.body) + .unwrap_or(session.transient_discontinuity_sequence.unwrap_or(0)); + session.transient_discontinuity_sequence = + Some(current_discontinuity_sequence.saturating_add(transient_visible_discontinuity_count(&rewritten.body))); + } else if let Some(discontinuity_sequence) = session.transient_discontinuity_sequence { + rewritten.body = apply_transient_discontinuity_sequence(&rewritten.body, discontinuity_sequence); + } + let manifest_validity = parse_manifest_validity(&rewritten.body); + session.transient.upsert_resources(rewritten.resources); + if let Some(validity) = manifest_validity { + session.transient.replace_manifest_with_validity(rewritten.body, rendered_at_ms, validity.playlist_duration_ms); + } else { + session.transient.replace_manifest(rewritten.body, rendered_at_ms); + } + let previous_highwater = session.origin_seq_highwater; + if let Some(highwater) = timeline.origin_highwater() { + session.origin_seq_highwater = + Some(next_committed_origin_highwater(session.origin_seq_highwater, highwater, quality.sequence_relation)); + } + + let timing = parse_manifest_timing(body); + if let Some(target_duration_ms) = timing.target_duration_ms { + if let Ok(target_duration_secs) = u32::try_from(target_duration_ms / 1_000) { + session.target_duration = Some(target_duration_secs); + } + } + let target_duration_ms = + timing.target_duration_ms.or_else(|| session.target_duration.map(|duration| u64::from(duration) * 1_000)); + let progress = + manifest_progress_from_highwater(previous_highwater, session.origin_seq_highwater, quality.sequence_relation); + build_manifest_refresh_timing(timing.last_segment_duration_ms, target_duration_ms, progress) +} + +fn commit_normal_manifest( + session: &mut super::HlsSession, + manifest: &ParsedOriginManifest, + _redirect_host: Option<&str>, + request: &OriginRefreshRequest, + _resolved_request_url: &str, + rendered_at_ms: u64, + sequence_relation: HlsManifestSequenceRelation, +) -> Result { + let previous_highwater = session.origin_seq_highwater; + let provisioning_handoff = session.pending_handoff_discontinuity_sequence.is_some() + && session.segments.values().any(is_hls_provisioning_segment); + let segment_durations = manifest.segments.iter().map(|segment| segment.duration_ms).collect::>(); + session.initial_prefetch_gap_segments = initial_hls_strip_segments_for_durations(&request.strip, &segment_durations); + session + .apply_origin_manifest(manifest) + .map_err(|err| HlsManifestCommitError::TimelineRejected { reason: HlsManifestRejectLogReason::from(err) })?; + if provisioning_handoff { + limit_publishable_normal_provisioning_handoff_tail(session, &request.strip, manifest.segments.len()); + } + session.origin_request_headers = request.headers.clone(); + session.queue_map_fetch_candidates(rendered_at_ms); + let backpressure = request.segment_worker_pool.classify_backpressure_for_session(session); + let queue_report = session.queue_manifest_fetch_candidates(rendered_at_ms, backpressure.allows_prefetch()); + request.segment_worker_pool.metrics().record_prefetch_queued(queue_report.prefetch_queued); + request.segment_worker_pool.metrics().record_prefetch_skipped(queue_report.prefetch_skipped); + if queue_report.prefetch_queued > 0 { + debug!( + "HLS segment queued for prefetch: session={} proxy_session_id={} count={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + queue_report.prefetch_queued + ); + } + if queue_report.prefetch_skipped > 0 { + debug!( + "HLS segment queued for prefetch skipped by backpressure: session={} proxy_session_id={} count={} state={backpressure:?}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + queue_report.prefetch_skipped + ); + } + match HlsManifestRenderer::render(session, rendered_at_ms) { + Ok(rendered) => { + let segment_count = rendered.segment_proxy_seqs.len(); + let render_gap_segments = rendered.render_gap_segments; + let media_sequence = rendered.first_proxy_seq; + match session.store_rendered_manifest(rendered) { + RenderedManifestStoreOutcome::Stored => { + request.segment_worker_pool.metrics().record_manifest_rendered(); + info!( + "HLS manifest rendered: session={} proxy_session_id={} media_sequence={} segments={} render_gap_segments={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + media_sequence, + segment_count, + render_gap_segments + ); + } + RenderedManifestStoreOutcome::Rejected( + RenderedManifestStoreRejectReason::RegressiveMediaSequence { + previous_first_proxy_seq, + candidate_first_proxy_seq, + }, + ) => { + request.segment_worker_pool.metrics().record_manifest_render_skipped(); + debug!( + "HLS manifest render rejected: session={} proxy_session_id={} reason=regressive-media-sequence previous={} candidate={}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id), + previous_first_proxy_seq, + candidate_first_proxy_seq + ); + } + } + } + Err(err) => { + request.segment_worker_pool.metrics().record_manifest_render_skipped(); + debug!( + "HLS manifest render skipped: session={} proxy_session_id={} reason={err:?}", + safe_session_key(&session.key), + safe_proxy_session_id(&session.proxy_session_id) + ); + } + } + + let last_segment_duration_ms = manifest.segments.last().map(|segment| segment.duration_ms); + let target_duration_ms = manifest.target_duration.map(|duration| u64::from(duration) * 1_000); + let progress = + manifest_progress_from_highwater(previous_highwater, session.origin_seq_highwater, sequence_relation); + Ok(build_manifest_refresh_timing(last_segment_duration_ms, target_duration_ms, progress)) +} + +fn limit_publishable_normal_provisioning_handoff_tail( + session: &mut super::HlsSession, + strip: &StripConfig, + origin_segment_count: usize, +) { + let Some(gap_seq) = session + .segments + .iter() + .filter_map(|(proxy_seq, entry)| is_hls_provisioning_gap_segment(entry).then_some(*proxy_seq)) + .max() + else { + return; + }; + let first_origin_seq = gap_seq.saturating_add(1); + if !session.segments.contains_key(&first_origin_seq) { + return; + } + if let Some(head_seq) = visible_provisioning_handoff_head_proxy_seq(session) { + session.publishable_origin_head_proxy_seq = Some(head_seq); + } + let initial_origin_segments = configured_handoff_origin_window_segments(strip, origin_segment_count).clamp(1, 3); + let tail_seq = + first_origin_seq.saturating_add(u64::try_from(initial_origin_segments.saturating_sub(1)).unwrap_or(0)); + if session.segments.contains_key(&tail_seq) { + session.publishable_origin_tail_proxy_seq = Some(tail_seq); + } +} + +fn visible_provisioning_handoff_head_proxy_seq(session: &super::HlsSession) -> Option { + session + .last_rendered_manifest + .as_ref() + .and_then(|rendered| { + rendered + .segment_proxy_seqs + .iter() + .copied() + .find(|proxy_seq| session.segments.get(proxy_seq).is_some_and(is_hls_provisioning_segment)) + }) + .or_else(|| { + session + .segments + .iter() + .filter_map(|(proxy_seq, entry)| is_hls_provisioning_segment(entry).then_some(*proxy_seq)) + .min() + }) +} + +fn configured_handoff_origin_window_segments(strip: &StripConfig, origin_segment_count: usize) -> usize { + match strip.mode { + HlsStripMode::Segments => usize::try_from(strip.value).unwrap_or(usize::MAX), + HlsStripMode::Seconds => 0, + } + .saturating_add(3) + .min(origin_segment_count) +} + +fn build_manifest_refresh_timing_base( + last_segment_duration_ms: Option, + target_duration_ms: Option, +) -> HlsManifestRefreshTiming { + let source = if last_segment_duration_ms.is_some() { + HlsManifestTimingSource::LastSegmentDuration + } else if target_duration_ms.is_some() { + HlsManifestTimingSource::TargetDuration + } else { + HlsManifestTimingSource::Fallback + }; + let base_interval_ms = compute_origin_refresh_interval_ms(last_segment_duration_ms, target_duration_ms); + HlsManifestRefreshTiming { + last_segment_duration_ms, + target_duration_ms, + base_interval_ms, + source, + progress: HlsManifestProgress::Unchanged, + } +} + +fn build_manifest_refresh_timing( + last_segment_duration_ms: Option, + target_duration_ms: Option, + progress: HlsManifestProgress, +) -> HlsManifestRefreshTiming { + let mut timing = build_manifest_refresh_timing_base(last_segment_duration_ms, target_duration_ms); + timing.progress = progress; + timing +} + +fn manifest_progress_from_highwater( + before: Option, + after: Option, + sequence_relation: HlsManifestSequenceRelation, +) -> HlsManifestProgress { + match (sequence_relation, before, after) { + (HlsManifestSequenceRelation::RolloverCandidate, _, _) => HlsManifestProgress::Rollover, + (_, None, Some(_)) => HlsManifestProgress::Advanced, + (_, Some(before), Some(after)) if after > before => HlsManifestProgress::Advanced, + _ => HlsManifestProgress::Unchanged, + } +} + +fn apply_empty_refresh_rampdown_ms(base_interval_ms: u64, empty_refresh_count: u32) -> u64 { + base_interval_ms.checked_shr(empty_refresh_count.min(16)).unwrap_or(0).max(1_000) +} + +fn log_manifest_refresh_timing( + session: &super::HlsSession, + timing: HlsManifestRefreshTiming, + refresh_interval_ms: u64, +) { + debug!( + "HLS manifest timing parsed: session={} target_duration={} last_segment_duration={} next_refresh_in_s={} source={} progress={} empty_refreshes={}", + safe_session_key(&session.key), + format_optional_millis_as_seconds(timing.target_duration_ms), + format_optional_millis_as_seconds(timing.last_segment_duration_ms), + format_millis_as_seconds(refresh_interval_ms), + timing.source.as_log_value(), + timing.progress.as_log_value(), + session.origin_refresh.consecutive_empty_refreshes + ); +} + +fn format_optional_millis_as_seconds(value_ms: Option) -> String { + value_ms.map_or_else(|| "none".to_string(), format_millis_as_seconds) +} + +fn format_millis_as_seconds(value_ms: u64) -> String { + let seconds = value_ms / 1_000; + let millis = value_ms % 1_000; + format!("{seconds}.{millis:03}") +} + +fn map_transient_reason(reason: OriginManifestTransientReason) -> TransientPassthroughReason { + match reason { + OriginManifestTransientReason::ExtXKey => TransientPassthroughReason::ExtXKey, + OriginManifestTransientReason::UnsupportedTag { tag } => TransientPassthroughReason::UnsupportedTag { tag }, + OriginManifestTransientReason::ParserUnsupportedFeature { feature } => { + TransientPassthroughReason::ParserUnsupportedFeature { feature } + } + } +} + +fn transient_reason_log_fields(reason: &TransientPassthroughReason) -> String { + match reason { + TransientPassthroughReason::ExtXKey => "reason=ext_x_key".to_string(), + TransientPassthroughReason::UnsupportedTag { tag } => format!("reason=unsupported_tag tag={tag}"), + TransientPassthroughReason::ParserUnsupportedFeature { feature } => { + format!("reason=parser_unsupported_feature feature={feature}") + } + } +} + +pub fn compute_origin_refresh_interval_ms( + last_segment_duration_ms: Option, + target_duration_ms: Option, +) -> u64 { + last_segment_duration_ms.or(target_duration_ms).map_or(2_000, |duration_ms| duration_ms / 2).max(1_000) +} + +pub fn cold_start_retry_after_seconds() -> u64 { COLD_START_RETRY_AFTER_SECONDS } + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{ + super::manifest_fetch::{ + classify_origin_manifest_status, hls_manifest_redirect_host, + manifest_host_switch_failure_threshold_for_strip_segments, origin_highwater_policy_limit, + refresh_from_live_hls_entrypoint_with_retries, resolved_hls_manifest_request_url_from_input, + retry_after_delay_ms, retry_hls_origin_manifest_recovery_chain, + score_hls_manifest_recovery_candidate as score_manifest_recovery_candidate, FetchedOriginManifest, + HlsManifestCommitAcceptanceMode, HlsManifestCommitError, HlsManifestOriginQualityScore, + HlsManifestRejectLogReason, HlsManifestSequenceRelation, LiveHlsOriginEntry, + ManifestRecoverySelectionLogPhase, OriginManifestFetchError, OriginManifestStatusClass, RetryPolicy, + }, + build_manifest_refresh_timing, commit_fetched_manifest, compute_origin_refresh_interval_ms, + fetched_effective_manifest_host, format_millis_as_seconds, format_optional_millis_as_seconds, + manifest_fetch_context, manifest_hard_fetch_error, manifest_progress_from_highwater, mark_origin_refresh_started, + manifest_temporary_failure_kind, record_pinned_host_recovery_chain_failed, request_error_indicates_timeout, + transient_reason_log_fields, HlsManifestAcceptanceDecision, HlsManifestCommitRequirement, HlsManifestProgress, + trigger_origin_refresh_sync, OriginRefreshRequest, OriginRefreshState, + }; + use crate::{ + api::model::{ + maybe_trigger_origin_refresh, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, + HlsFreshManifestRequiredReason, HlsMapWorkerPool, HlsOriginAccountBinding, HlsPlaybackFamilyKey, + HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSession, + HlsSessionKey, HlsSessionMode, HlsBoundAccountAcquireErrorKind, TimelineMapError, + TransientPassthroughReason, + }, + model::{ + AppConfig, Config, ConfigProvider, HlsManifestRecoveryBurstConfig, + HlsSegmentRepairConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, + }, + }; + use arc_swap::{ArcSwap, ArcSwapOption}; + use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}; + use shared::model::{ConfigPaths, ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, ProviderUrlSelectionPolicy}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{Mutex, RwLock}, + }; + use url::Url; + + async fn retry_test_manifest_recovery_chain( + request: &OriginRefreshRequest, + target_url: Url, + reject_reason: HlsManifestRejectLogReason, + ) -> Result { + let fetch_context = manifest_fetch_context(request); + retry_hls_origin_manifest_recovery_chain( + &fetch_context, + target_url, + None, + Some(reject_reason), + |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(request, fetched, acceptance_mode), + ) + .await + } + + fn test_session() -> Arc> { + Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))) + } + + fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) + } + + fn test_app_config() -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::empty()), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), + }) + } + + #[test] + fn origin_refresh_state_starts_only_when_due_and_not_in_flight() { + let mut state = OriginRefreshState { next_fetch_allowed_at_ms: 100, ..OriginRefreshState::default() }; + assert!(!state.is_due(99)); + assert!(state.is_due(100)); + state.mark_started(100); + assert!(!state.is_due(101)); + } + + #[test] + fn origin_refresh_failure_backoff_ramps_and_success_resets_counter() { + let mut state = OriginRefreshState::default(); + + state.mark_started(1_000); + state.mark_failure(1_100); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.last_error_at_ms, Some(1_100)); + assert_eq!(state.next_fetch_allowed_at_ms, 1_100); + assert!(state.is_due(1_100)); + + state.mark_started(1_200); + state.mark_failure(1_300); + assert_eq!(state.consecutive_failures, 2); + assert_eq!(state.next_fetch_allowed_at_ms, 1_800); + assert!(!state.is_due(1_799)); + assert!(state.is_due(1_800)); + + state.mark_started(1_900); + state.mark_failure(2_000); + assert_eq!(state.consecutive_failures, 3); + assert_eq!(state.next_fetch_allowed_at_ms, 3_000); + + state.mark_started(3_100); + let success_timing = build_manifest_refresh_timing(Some(20_000), None, HlsManifestProgress::Advanced); + assert_eq!(state.mark_success_with_timing(3_100, 3_200, success_timing), 10_000); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.last_error_at_ms, None); + assert_eq!(state.next_fetch_allowed_at_ms, 13_100); + + state.mark_started(13_100); + state.mark_failure(13_200); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.next_fetch_allowed_at_ms, 13_200); + } + + #[test] + fn status_classification_matches_hls_retry_policy() { + for status in [ + StatusCode::PROXY_AUTHENTICATION_REQUIRED, + StatusCode::REQUEST_TIMEOUT, + StatusCode::TOO_EARLY, + StatusCode::TOO_MANY_REQUESTS, + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::BAD_GATEWAY, + ] { + assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::Retryable); + } + for status in [ + StatusCode::BAD_REQUEST, + StatusCode::UNAUTHORIZED, + StatusCode::FORBIDDEN, + StatusCode::NOT_FOUND, + StatusCode::GONE, + ] { + assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::PermanentFailure); + } + } + + #[test] + fn manifest_temporary_failure_kind_counts_retryable_status_and_timeout_only() { + assert_eq!( + manifest_temporary_failure_kind(&OriginManifestFetchError::RetryableStatus( + StatusCode::TOO_MANY_REQUESTS, + None, + )), + Some(crate::api::model::HlsManifestTemporaryFailureKind::RetryableStatus { + status: StatusCode::TOO_MANY_REQUESTS + }) + ); + assert_eq!( + manifest_temporary_failure_kind(&OriginManifestFetchError::Timeout), + Some(crate::api::model::HlsManifestTemporaryFailureKind::Timeout) + ); + assert_eq!( + manifest_temporary_failure_kind(&OriginManifestFetchError::Request( + "Request timed out and no retries left".to_string(), + )), + Some(crate::api::model::HlsManifestTemporaryFailureKind::Timeout) + ); + assert_eq!( + manifest_temporary_failure_kind(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::WaitTimedOut, + )), + Some(crate::api::model::HlsManifestTemporaryFailureKind::ProviderAcquire { + kind: HlsBoundAccountAcquireErrorKind::WaitTimedOut + }) + ); + assert!(manifest_temporary_failure_kind(&OriginManifestFetchError::Request( + "Request error: error sending request".to_string(), + )) + .is_none()); + assert!(manifest_temporary_failure_kind(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::Expired, + )) + .is_none()); + assert!(manifest_temporary_failure_kind(&OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND)) + .is_none()); + } + + #[test] + fn manifest_hard_fetch_error_matches_permanent_and_non_retryable_status_only() { + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::NonRetryableStatus(StatusCode::IM_A_TEAPOT))); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::RetryableStatus( + StatusCode::TOO_MANY_REQUESTS, + None, + ))); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::Timeout)); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::WaitTimedOut, + ))); + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::Expired, + ))); + } + + #[test] + fn request_error_timeout_detection_matches_global_helper_wording() { + assert!(request_error_indicates_timeout("Request timed out and no retries left")); + assert!(request_error_indicates_timeout("idle timeout while trying provider://demo")); + assert!(!request_error_indicates_timeout("Request error: error sending request")); + } + + #[test] + fn manifest_reject_log_reason_formats_host_switch_pending() { + let reason = HlsManifestRejectLogReason::HostSwitchPending { failures: 2, threshold: 3 }; + assert_eq!(reason.status_label(), "host-switch-pending failures=2 threshold=3"); + } + + #[test] + fn manifest_reject_log_reason_preserves_timeline_mapping_error() { + assert_eq!( + HlsManifestRejectLogReason::from(TimelineMapError::UnsupportedSegmentExtension).status_label(), + "unsupported-segment-extension" + ); + assert_eq!( + HlsManifestRejectLogReason::from(TimelineMapError::ProxyMapIdOverflow).status_label(), + "proxy-map-id-overflow" + ); + } + + #[test] + fn manifest_host_switch_failure_threshold_uses_half_of_visible_window_with_cap() { + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(0), 1); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(1), 2); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(2), 2); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(3), 3); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(6), 4); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(7), 5); + assert_eq!(manifest_host_switch_failure_threshold_for_strip_segments(100), 5); + } + + #[test] + fn manifest_highwater_policy_limit_uses_target_duration_fallback() { + assert_eq!(origin_highwater_policy_limit(60, None), Some(4)); + assert_eq!(origin_highwater_policy_limit(61, None), Some(5)); + assert_eq!(origin_highwater_policy_limit(60, Some(12)), Some(5)); + } + + #[test] + fn manifest_recovery_burst_levels_map_to_candidate_counts() { + let cases = [ + (HlsManifestRecoveryBurstLevel::Off, 1, 1, 1), + (HlsManifestRecoveryBurstLevel::Friendly, 2, 1, 2), + (HlsManifestRecoveryBurstLevel::Cautious, 3, 1, 3), + (HlsManifestRecoveryBurstLevel::Balanced, 4, 1, 4), + (HlsManifestRecoveryBurstLevel::Intense, 5, 1, 5), + (HlsManifestRecoveryBurstLevel::Aggressive, 6, 1, 6), + (HlsManifestRecoveryBurstLevel::Beast, 6, 2, 12), + ]; + for (level, expected_slots, expected_lanes, expected_candidates) in cases { + let plan = level.plan(); + assert_eq!(plan.slots, expected_slots); + assert_eq!(plan.lanes_per_slot, expected_lanes); + assert_eq!(plan.total_candidates(), expected_candidates); + assert_eq!(level.total_candidates(), expected_candidates); + } + } + + #[test] + fn retry_after_header_is_parsed_as_milliseconds() { + let mut headers = HeaderMap::new(); + headers.insert(header::RETRY_AFTER, HeaderValue::from_static("3")); + assert_eq!(retry_after_delay_ms(&headers), Some(3_000)); + } + + #[test] + fn resolved_hls_manifest_request_url_uses_provider_index_locally() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + })); + let provider_entry = + LiveHlsOriginEntry::parse_with_url_failover_provider("provider://demo/live/u/p/1.m3u8", Some(provider)) + .unwrap(); + + let resolved = resolved_hls_manifest_request_url_from_input( + &provider_entry.to_input_source(), + Some(1), + provider_entry.url(), + ); + assert_eq!(resolved.as_str(), "http://provider-b.example/live/u/p/1.m3u8"); + assert!(!resolved.as_str().contains("provider://")); + + let direct_entry = LiveHlsOriginEntry::parse("http://origin.example/live/u/p/1.m3u8").unwrap(); + assert_eq!( + resolved_hls_manifest_request_url_from_input(&direct_entry.to_input_source(), Some(1), direct_entry.url()) + .as_str(), + "http://origin.example/live/u/p/1.m3u8" + ); + } + + #[test] + fn manifest_timing_log_values_are_seconds_or_none() { + assert_eq!(format_optional_millis_as_seconds(Some(4_500)), "4.500"); + assert_eq!(format_optional_millis_as_seconds(None), "none"); + assert_eq!(format_millis_as_seconds(2_000), "2.000"); + } + + #[test] + fn refresh_interval_uses_half_reference_duration_without_upper_clamp() { + assert_eq!(compute_origin_refresh_interval_ms(Some(8_000), None), 4_000); + assert_eq!(compute_origin_refresh_interval_ms(Some(500), None), 1_000); + assert_eq!(compute_origin_refresh_interval_ms(Some(20_000), None), 10_000); + assert_eq!(compute_origin_refresh_interval_ms(None, None), 2_000); + } + + #[test] + fn empty_refresh_rampdown_halves_until_one_second() { + let mut state = OriginRefreshState::default(); + let timing = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, timing), 3_000); + assert_eq!(state.consecutive_empty_refreshes, 1); + assert_eq!(state.next_fetch_allowed_at_ms, 3_000); + + state.mark_started(3_000); + assert_eq!(state.mark_success_with_timing(3_000, 3_100, timing), 1_500); + assert_eq!(state.consecutive_empty_refreshes, 2); + assert_eq!(state.next_fetch_allowed_at_ms, 4_500); + + state.mark_started(4_500); + assert_eq!(state.mark_success_with_timing(4_500, 4_600, timing), 1_000); + assert_eq!(state.consecutive_empty_refreshes, 3); + assert_eq!(state.next_fetch_allowed_at_ms, 5_500); + + state.mark_started(5_500); + assert_eq!(state.mark_success_with_timing(5_500, 5_600, timing), 1_000); + assert_eq!(state.consecutive_empty_refreshes, 4); + assert_eq!(state.next_fetch_allowed_at_ms, 6_500); + } + + #[test] + fn advanced_or_rollover_refresh_resets_empty_refresh_counter() { + let mut state = OriginRefreshState::default(); + let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + let advanced = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Advanced); + let rollover = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Rollover); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); + state.mark_started(3_000); + assert_eq!(state.mark_success_with_timing(3_000, 3_100, unchanged), 1_500); + assert_eq!(state.consecutive_empty_refreshes, 2); + + state.mark_started(4_500); + assert_eq!(state.mark_success_with_timing(4_500, 4_600, advanced), 6_000); + assert_eq!(state.consecutive_empty_refreshes, 0); + assert_eq!(state.next_fetch_allowed_at_ms, 10_500); + + state.mark_started(10_500); + assert_eq!(state.mark_success_with_timing(10_500, 10_600, unchanged), 3_000); + assert_eq!(state.consecutive_empty_refreshes, 1); + + state.mark_started(13_500); + assert_eq!(state.mark_success_with_timing(13_500, 13_600, rollover), 6_000); + assert_eq!(state.consecutive_empty_refreshes, 0); + assert_eq!(state.next_fetch_allowed_at_ms, 19_500); + } + + #[test] + fn failure_backoff_does_not_increment_empty_refresh_counter() { + let mut state = OriginRefreshState::default(); + let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); + state.mark_started(3_000); + state.mark_failure(3_100); + + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.consecutive_empty_refreshes, 1); + } + + #[test] + fn manifest_progress_tracks_highwater_advancement() { + assert_eq!( + manifest_progress_from_highwater(None, Some(10), HlsManifestSequenceRelation::NoPreviousHighwater), + HlsManifestProgress::Advanced + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(11), HlsManifestSequenceRelation::Next), + HlsManifestProgress::Advanced + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(10), HlsManifestSequenceRelation::Same), + HlsManifestProgress::Unchanged + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(9), HlsManifestSequenceRelation::Backward), + HlsManifestProgress::Unchanged + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(1), HlsManifestSequenceRelation::RolloverCandidate), + HlsManifestProgress::Rollover + ); + assert_eq!( + manifest_progress_from_highwater(None, None, HlsManifestSequenceRelation::NoOriginHighwater), + HlsManifestProgress::Unchanged + ); + } + + #[test] + fn recovery_selection_log_phase_distinguishes_single_candidate_from_burst() { + assert_eq!( + ManifestRecoverySelectionLogPhase::from_candidate_count(1), + ManifestRecoverySelectionLogPhase::Recovery + ); + assert_eq!( + ManifestRecoverySelectionLogPhase::from_candidate_count(2), + ManifestRecoverySelectionLogPhase::Burst + ); + assert_eq!(ManifestRecoverySelectionLogPhase::Recovery.as_log_label(), "recovery"); + assert_eq!(ManifestRecoverySelectionLogPhase::Burst.as_log_label(), "burst"); + } + + #[test] + fn transient_reason_log_fields_include_unsupported_tag() { + let reason = TransientPassthroughReason::UnsupportedTag { tag: "#EXT-X-PART".to_string() }; + + assert_eq!(transient_reason_log_fields(&reason), "reason=unsupported_tag tag=#EXT-X-PART"); + } + + #[test] + fn different_host_candidate_is_not_committed_immediately() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); + let candidate = session.manifest_acceptance.host_switch_candidate.as_ref().expect("candidate is held"); + assert_eq!(candidate.host, "origin.example.com"); + assert_eq!(candidate.highwater, Some(759)); + assert!(session.transient.last_manifest_body.is_none()); + } + + #[test] + fn fresh_commit_rebases_normal_manifest_against_stale_session_baseline() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.manifest_acceptance.same_host_retry_chain_failures = 3; + session.manifest_acceptance.host_switch_candidate = Some(super::super::HlsManifestHostSwitchCandidate { + host: "stale.example.com".to_string(), + target_url: "http://stale.example.com/live/user/pass/12345.m3u8".to_string(), + first_seen_at_ms: 1, + last_seen_at_ms: 2, + seen_count: 2, + highwater: Some(1_000), + quality_score: 1, + }); + let mut request = test_origin_refresh_request(test_session()); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + let fetched = + fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg10.ts\n"); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); + assert_eq!(session.manifest_acceptance.same_host_retry_chain_failures, 0); + assert!(session.manifest_acceptance.host_switch_candidate.is_none()); + } + + #[test] + fn host_switch_candidate_commits_only_after_failed_chain_threshold() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", + ); + + assert!(matches!( + commit_fetched_manifest(&mut session, &fetched, &request, 100), + Err(HlsManifestCommitError::RetryCurrentTarget) + )); + assert!(matches!( + record_pinned_host_recovery_chain_failed( + &mut session, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + 200 + ), + HlsManifestAcceptanceDecision::Reject { .. } + )); + assert!(matches!( + record_pinned_host_recovery_chain_failed( + &mut session, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + 300 + ), + HlsManifestAcceptanceDecision::Reject { .. } + )); + assert!(matches!( + record_pinned_host_recovery_chain_failed( + &mut session, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + 400 + ), + HlsManifestAcceptanceDecision::AcceptHostSwitch { .. } + )); + + let result = super::commit_fetched_manifest_with_acceptance_mode( + &mut session, + &fetched, + &request, + 500, + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(759)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); + assert_eq!(session.manifest_acceptance.same_host_retry_chain_failures, 0); + assert!(session.manifest_acceptance.host_switch_candidate.is_none()); + } + + #[tokio::test] + async fn concurrent_maybe_trigger_origin_refresh_starts_singleflight_once() { + let session = test_session(); + let entry = + LiveHlsOriginEntry::parse("http://127.0.0.1:9/live/user/pass/12345.m3u8").expect("valid origin entry"); + let client = reqwest::Client::new(); + let no_redirect_client = + reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client, + no_redirect_client, + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 1, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 }, + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + let mut handles = Vec::new(); + for _ in 0..8 { + let request = request.clone(); + handles.push(tokio::spawn(async move { maybe_trigger_origin_refresh(request).await })); + } + + let started = futures::future::join_all(handles) + .await + .into_iter() + .filter(|result| result.as_ref().is_ok_and(|started| *started)) + .count(); + assert_eq!(started, 1); + } + + #[tokio::test] + async fn fresh_manifest_commit_bypasses_refresh_debounce() { + let session = test_session(); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ColdStart, + }; + + assert!(mark_origin_refresh_started(&mut request, 1_000).await); + assert!(session.read().await.origin_refresh.in_flight); + } + + #[tokio::test] + async fn committed_manifest_refresh_still_obeys_debounce() { + let session = test_session(); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; + let mut request = test_origin_refresh_request(Arc::clone(&session)); + + assert!(!mark_origin_refresh_started(&mut request, 1_000).await); + assert!(!session.read().await.origin_refresh.in_flight); + } + + #[tokio::test] + async fn successful_manifest_commit_shortens_pending_leases_without_response_path() { + let session = test_session(); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let hls_proxy = Arc::new(HlsProxyManager::new()); + let now_ms = super::current_time_millis(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("user", "client"), + proxy_session_id.clone(), + "user".to_string(), + "session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 90_000, + )) + .await; + let server = spawn_test_origin(Arc::new(|_path| { + (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.origin_entry = entry; + request.now_ms = now_ms; + + assert!(trigger_origin_refresh_sync(request).await); + + let lease = hls_proxy + .access_leases() + .write() + .await + .response_snapshot(&lease_id, &proxy_session_id, super::current_time_millis()) + .expect("pending lease should remain available"); + let Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms }) = lease.pending_deadline else { + panic!("pending lease should be shortened to follow-up"); + }; + assert!(deadline_ms < now_ms.saturating_add(90_000)); + assert!(deadline_ms <= super::current_time_millis().saturating_add(10_000)); + } + + async fn refresh_session_with_origin_body(body: &'static str) -> Arc> { + let session = test_session(); + let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), body.to_string()))).await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.transient.last_manifest_body.is_some() { + return session; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + session + } + + #[tokio::test] + async fn refresh_stores_headers_after_hls_origin_policy() { + let session = test_session(); + let server = spawn_test_origin(Arc::new(|_path| { + (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("blocked")); + headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); + headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers, + origin_provider_session_headers: HeaderMap::new(), + disabled_headers: Some(ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: true, + cloudflare_header: true, + custom_header: Vec::new(), + }), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.last_rendered_manifest.is_some() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let session = session.read().await; + assert!(!session.origin_request_headers.contains_key(header::AUTHORIZATION)); + assert!(!session.origin_request_headers.contains_key(header::COOKIE)); + assert!(!session.origin_request_headers.contains_key("proxy-authorization")); + assert!(!session.origin_request_headers.contains_key(header::HOST)); + assert!(!session.origin_request_headers.contains_key("x-blocked")); + assert!(!session.origin_request_headers.contains_key("cf-ray")); + assert_eq!(session.origin_request_headers.get(header::ACCEPT_LANGUAGE).expect("language"), "de"); + } + + #[tokio::test] + async fn accepted_manifest_commit_stores_provider_session_cookie_separately() { + let session = test_session(); + let server = spawn_test_origin(Arc::new(|_path| { + ( + 200, + vec![ + ("Set-Cookie", "sid=abc; Path=/; HttpOnly".to_string()), + ("Set-Cookie", "pref=1; SameSite=Lax".to_string()), + ], + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string(), + ) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + + assert!(trigger_origin_refresh_sync(request).await); + + let session = session.read().await; + assert!(!session.origin_request_headers.contains_key(header::COOKIE)); + assert_eq!( + session.origin_provider_session_headers.get(header::COOKIE).expect("provider cookie"), + "sid=abc; pref=1" + ); + } + + #[test] + fn origin_account_binding_change_clears_provider_session_headers() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "1"), b"secret", 100); + session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=abc")); + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-a"), + &session.proxy_session_id.clone(), + 100, + ))); + assert!(session.origin_provider_session_headers.is_empty()); + + session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=next")); + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-a"), + &session.proxy_session_id.clone(), + 200, + ))); + assert!(!session.origin_provider_session_headers.is_empty()); + + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-b"), + &session.proxy_session_id.clone(), + 300, + ))); + assert!(session.origin_provider_session_headers.is_empty()); + } + + #[tokio::test] + async fn ext_x_key_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ) + .await; + let session = session.read().await; + let body = session.transient.last_manifest_body.as_ref().expect("transient body"); + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey } + )); + assert!(body.contains("/hls/shared/live/")); + assert!(body.contains("/r/")); + assert!(!body.contains("/hls/user/")); + assert_eq!(session.transient.resources.len(), 2); + assert_eq!(session.target_duration, Some(12)); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); + } + + #[test] + fn transient_commit_accepts_plausible_same_redirect_host_rollover_and_resets_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(100); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest(previous_manifest.clone(), 10); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(0)); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); + } + + #[test] + fn transient_commit_rejects_same_host_backward_manifest_outside_rollover_window() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(100); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest(previous_manifest.clone(), 10); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:226\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::TimelineRejected { .. }))); + assert_eq!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(758)); + } + + #[test] + fn transient_commit_rebases_expired_session_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.target_duration = Some(12); + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(1_000); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest(previous_manifest.clone(), 10); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 40_000); + + assert!(result.is_ok()); + assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(900)); + } + + #[test] + fn transient_commit_accepts_monotonic_media_sequence_and_updates_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.transient.replace_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(), + 10, + ); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:759\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg759.ts\n#EXTINF:4.0,\nseg760.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(760)); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); + } + + #[test] + fn transient_commit_with_different_redirect_host_is_held_as_candidate() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); + assert_eq!(session.origin_seq_highwater, Some(758)); + assert!(session.transient.last_manifest_body.is_none()); + let candidate = session.manifest_acceptance.host_switch_candidate.as_ref().expect("candidate"); + assert_eq!(candidate.host, "origin.example.com"); + assert_eq!(candidate.highwater, Some(759)); + } + + #[test] + fn fresh_commit_rebases_transient_manifest_against_stale_session_baseline() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.manifest_acceptance.same_host_retry_chain_failures = 3; + session.manifest_acceptance.host_switch_candidate = Some(super::super::HlsManifestHostSwitchCandidate { + host: "stale.example.com".to_string(), + target_url: "http://stale.example.com/live/user/pass/12345.m3u8".to_string(), + first_seen_at_ms: 1, + last_seen_at_ms: 2, + seen_count: 2, + highwater: Some(1_000), + quality_score: 1, + }); + let mut request = test_origin_refresh_request(test_session()); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg10.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); + assert_eq!(session.manifest_acceptance.same_host_retry_chain_failures, 0); + assert!(session.manifest_acceptance.host_switch_candidate.is_none()); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); + } + + #[tokio::test] + async fn unsupported_tag_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body("#EXTM3U\n#EXT-X-PART:DURATION=1.0,URI=\"part.m4s\"\n").await; + let session = session.read().await; + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::UnsupportedTag { .. } } + )); + assert!(session.transient.last_manifest_body.is_some()); + } + + #[tokio::test] + async fn parser_unsupported_feature_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body("#EXTM3U\n#EXT-X-BYTERANGE:10\n#EXTINF:4.0,\nseg.ts\n").await; + let session = session.read().await; + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { + reason: TransientPassthroughReason::ParserUnsupportedFeature { .. } + } + )); + assert!(session.transient.last_manifest_body.is_some()); + } + + struct TestOriginServer { + base_url: String, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + } + + type TestOriginHandler = Arc (u16, Vec<(&'static str, String)>, String) + Send + Sync>; + + impl Drop for TestOriginServer { + fn drop(&mut self) { self.task.abort(); } + } + + async fn spawn_test_origin(handler: TestOriginHandler) -> TestOriginServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_task = Arc::clone(&requests); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&requests_for_task); + let handler = Arc::clone(&handler); + tokio::spawn(async move { + let mut buf = vec![0_u8; 4096]; + let mut used = 0_usize; + loop { + let Ok(read) = socket.read(&mut buf[used..]).await else { + return; + }; + if read == 0 { + return; + } + used += read; + if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if used == buf.len() { + return; + } + } + let request = String::from_utf8_lossy(&buf[..used]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/") + .to_string(); + requests.lock().await.push(path.clone()); + let (status, headers, body) = handler(path); + let reason = match status { + 200 => "OK", + 302 => "Found", + 404 => "Not Found", + 407 => "Proxy Authentication Required", + 500 => "Internal Server Error", + _ => "Status", + }; + let mut response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + for (name, value) in headers { + response.push_str(name); + response.push_str(": "); + response.push_str(&value); + response.push_str("\r\n"); + } + response.push_str("\r\n"); + response.push_str(&body); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + TestOriginServer { base_url: format!("http://{addr}"), requests, task } + } + + fn no_delay_policy() -> RetryPolicy { RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 } } + + fn test_origin_refresh_request(session: Arc>) -> OriginRefreshRequest { + let entry = LiveHlsOriginEntry::parse("http://origin.example.com/live/user/pass/12345.m3u8") + .expect("valid origin entry"); + OriginRefreshRequest { + app_config: test_app_config(), + session, + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + } + } + + fn fetched_manifest(body: &str) -> FetchedOriginManifest { + FetchedOriginManifest { + body: body.to_string(), + final_manifest_url: "http://origin.example.com/live/final/index.m3u8".to_string(), + resolved_request_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), + redirect_host: Some("origin.example.com".to_string()), + provider_url_index: None, + provider_session_headers: HeaderMap::new(), + status: StatusCode::OK, + attempts: 1, + } + } + + #[test] + fn provider_failover_mirror_without_redirect_uses_resolved_host_signal() { + let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); + fetched.redirect_host = None; + fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); + fetched.provider_url_index = Some(1); + + assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("mirror.example.com")); + } + + #[test] + fn provider_failover_with_redirect_uses_redirect_host_as_manifest_host_signal() { + let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); + fetched.redirect_host = Some("redirect.example.com".to_string()); + fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); + fetched.provider_url_index = Some(1); + + assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("redirect.example.com")); + } + + #[test] + fn manifest_redirect_host_is_only_set_for_actual_redirect_host_switch() { + let resolved = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("resolved url"); + let same_target = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("same url"); + let redirected = Url::parse("http://cdn.example.net/live/play/12345.m3u8").expect("redirect url"); + + assert_eq!(hls_manifest_redirect_host(&resolved, &same_target), None); + assert_eq!(hls_manifest_redirect_host(&resolved, &redirected).as_deref(), Some("cdn.example.net")); + } + + fn host_from_base_url(base_url: &str) -> String { + url::Url::parse(base_url).expect("base url").host_str().expect("host").to_string() + } + + fn manifest_body() -> String { "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string() } + + #[tokio::test] + async fn manifest_retry_starts_at_entrypoint_after_redirect_failure() { + let redirect_hits = Arc::new(AtomicUsize::new(0)); + let redirect_hits_for_handler = Arc::clone(&redirect_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path == "/live/user/pass/12345.m3u8" { + return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); + } + if path == "/live/play/once/12345" { + let hit = redirect_hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit < 2 { + return (500, Vec::new(), "fail".to_string()); + } + return (200, Vec::new(), manifest_body()); + } + (404, Vec::new(), String::new()) + })) + .await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let no_redirect_client = + reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); + + let fetched = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &no_redirect_client, + true, + 2_000, + &no_delay_policy(), + ) + .await + .expect("refresh eventually succeeds"); + + assert_eq!(fetched.attempts, 3); + assert_eq!( + *server.requests.lock().await, + vec![ + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345" + ] + ); + } + + #[tokio::test] + async fn retryable_407_retries_until_success() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit < 2 { + return (407, Vec::new(), "retry".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + + let fetched = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &reqwest::Client::new(), + false, + 2_000, + &no_delay_policy(), + ) + .await + .expect("refresh eventually succeeds"); + + assert_eq!(fetched.attempts, 3); + assert_eq!(server.requests.lock().await.len(), 3); + } + + #[tokio::test] + async fn permanent_404_does_not_retry() { + let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + + let err = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &reqwest::Client::new(), + false, + 2_000, + &no_delay_policy(), + ) + .await + .expect_err("404 is permanent"); + + assert!(matches!(err, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + assert_eq!(server.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn provider_failover_status_does_not_count_as_hls_retry() { + let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; + let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + let initial_session_key = session.read().await.key.stable_value(); + let initial_proxy_session_id = session.read().await.proxy_session_id.clone(); + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let segment_worker_pool = Arc::new(HlsSegmentWorkerPool::default()); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::clone(&segment_worker_pool), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.origin_seq_highwater == Some(102) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let first_requests = first.requests.lock().await; + let second_requests = second.requests.lock().await; + let first_manifest_requests = + first_requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + let second_manifest_requests = + second_requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(first_manifest_requests, 1); + assert_eq!(second_manifest_requests, 1); + let session = session.read().await; + assert_eq!(session.key.stable_value(), initial_session_key); + assert_eq!(session.proxy_session_id, initial_proxy_session_id); + assert!(!session.key.stable_value().contains("provider://")); + assert!(!session.key.stable_value().contains(first.base_url.as_str())); + assert!(!session.key.stable_value().contains(second.base_url.as_str())); + assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); + assert!(session.manifest_acceptance.host_switch_candidate.is_none()); + let metrics = segment_worker_pool.metrics().snapshot(); + assert_eq!(metrics.refresh_started, 1); + assert_eq!(metrics.refresh_completed, 1); + assert_eq!(metrics.refresh_retried, 0); + assert_eq!(metrics.refresh_failed, 0); + } + + #[tokio::test] + async fn different_host_retries_current_target_before_switching() { + let candidate_hits = Arc::new(AtomicUsize::new(0)); + let candidate_hits_for_handler = Arc::clone(&candidate_hits); + let candidate = spawn_test_origin(Arc::new(move |_path| { + candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + } + let candidate_entry_url = + format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); + let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.origin_seq_highwater == Some(102) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let candidate_requests = + candidate.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(candidate_requests, 6); + assert_eq!(session.read().await.origin_seq_highwater, Some(102)); + assert_eq!(session.read().await.manifest_acceptance.same_host_retry_chain_failures, 0); + } + + #[tokio::test] + async fn manifest_recovery_burst_skips_rejected_candidate() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:50\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n50.bin\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); + } + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = + HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); + let committed = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await + .expect("burst should commit accepted candidate"); + + assert_eq!(committed.fetched.attempts, 1); + assert_eq!(hits.load(Ordering::SeqCst), 2); + assert_eq!(session.read().await.origin_seq_highwater, Some(101)); + } + + #[test] + fn manifest_recovery_candidate_score_prefers_same_host_next_sequence() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(super::current_time_millis()); + let request = test_origin_refresh_request(test_session()); + let fetch_context = manifest_fetch_context(&request); + let same_host_unchanged = + fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n"); + let same_host_next = + fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n"); + let mut other_host_next = same_host_next.clone(); + other_host_next.redirect_host = Some("other.example.com".to_string()); + + assert_eq!( + score_manifest_recovery_candidate(&session, &same_host_unchanged, &fetch_context) + .expect("score") + .quality + .score, + HlsManifestOriginQualityScore::SameHostUnchanged + ); + assert_eq!( + score_manifest_recovery_candidate(&session, &same_host_next, &fetch_context).expect("score").quality.score, + HlsManifestOriginQualityScore::SameHostNextSequence + ); + let other_host_score = score_manifest_recovery_candidate(&session, &other_host_next, &fetch_context) + .expect("score") + .quality; + assert_eq!(other_host_score.score, HlsManifestOriginQualityScore::OtherHostNextSequence); + assert!(other_host_score.requires_handoff_discontinuity); + } + + #[tokio::test] + async fn manifest_recovery_burst_commits_best_same_host_candidate() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); + } + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = + HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); + let committed = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await + .expect("burst should commit best same-host candidate"); + + assert_eq!(committed.fetched.attempts, 1); + assert_eq!(hits.load(Ordering::SeqCst), 2); + assert_eq!(session.read().await.origin_seq_highwater, Some(101)); + } + + #[tokio::test] + async fn provider_failover_initial_success_commits_without_hls_host_retry_when_unpinned() { + let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; + let second_hits = Arc::new(AtomicUsize::new(0)); + let second_hits_for_handler = Arc::clone(&second_hits); + let second = spawn_test_origin(Arc::new(move |_path| { + let hit = second_hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:102\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n102.ts\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n".to_string(), + ) + })) + .await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + { + session.write().await.origin_seq_highwater = Some(100); + } + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.origin_seq_highwater == Some(102) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let first_manifest_requests = + first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + let second_manifest_requests = + second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(first_manifest_requests, 1); + assert_eq!(second_manifest_requests, 1); + assert_eq!(session.read().await.origin_seq_highwater, Some(102)); + } + + #[tokio::test] + async fn host_switch_failure_counter_increments_once_per_full_retry_chain() { + let candidate_hits = Arc::new(AtomicUsize::new(0)); + let candidate_hits_for_handler = Arc::clone(&candidate_hits); + let candidate = spawn_test_origin(Arc::new(move |_path| { + candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n900.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + } + let candidate_entry_url = + format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); + let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if candidate_hits.load(Ordering::SeqCst) >= 6 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(candidate_hits.load(Ordering::SeqCst), 6); + assert_eq!(session.read().await.origin_seq_highwater, Some(100)); + assert_eq!(session.read().await.manifest_acceptance.same_host_retry_chain_failures, 1); + } +} diff --git a/backend/src/api/model/hls_cache/renderer.rs b/backend/src/api/model/hls_cache/renderer.rs new file mode 100644 index 000000000..dadc09a4a --- /dev/null +++ b/backend/src/api/model/hls_cache/renderer.rs @@ -0,0 +1,765 @@ +use super::{ + is_hls_provisioning_gap_segment, is_hls_provisioning_segment, HlsSession, MapCacheStatus, ProxySessionId, + SegmentCacheStatus, SegmentEntry, HLS_ACCESS_LEASE_ID_PLACEHOLDER, HLS_PROVISIONING_TARGET_DURATION_SECS, +}; +use crate::model::StripConfig; +use shared::model::HlsStripMode; +use std::fmt::Write as _; + +const MIN_VISIBLE_SEGMENTS: usize = 3; +const TARGET_VISIBLE_SEGMENTS: usize = 6; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct RenderPolicy { + pub initial_render_gap_segments: usize, + pub max_not_ready_render_segments: usize, +} + +impl RenderPolicy { + pub fn new(initial_render_gap_segments: usize) -> Self { + Self { initial_render_gap_segments, max_not_ready_render_segments: 2 } + } + + pub fn from_strip_config(strip: &StripConfig, segment_durations_ms: &[u64]) -> Self { + match strip.mode { + HlsStripMode::Segments => Self::new(usize::try_from(strip.value).unwrap_or(usize::MAX)), + HlsStripMode::Seconds => { + let target_ms = strip.value.saturating_mul(1_000); + let mut accumulated_ms = 0_u64; + let mut gap_segments = 0_usize; + for duration_ms in segment_durations_ms.iter().rev() { + if accumulated_ms >= target_ms { + break; + } + accumulated_ms = accumulated_ms.saturating_add(*duration_ms); + gap_segments = gap_segments.saturating_add(1); + } + Self::new(gap_segments) + } + } + } +} + +impl Default for RenderPolicy { + fn default() -> Self { Self::new(0) } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct RenderedManifest { + pub body: String, + pub first_proxy_seq: u64, + pub last_proxy_seq: u64, + pub playlist_duration_ms: u64, + pub valid_until_ms: u64, + pub render_gap_segments: usize, + pub rendered_at_ms: u64, + pub segment_proxy_seqs: Vec, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RenderedManifestStoreOutcome { + Stored, + Rejected(RenderedManifestStoreRejectReason), +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RenderedManifestStoreRejectReason { + RegressiveMediaSequence { + previous_first_proxy_seq: u64, + candidate_first_proxy_seq: u64, + }, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum RenderError { + NoRenderableWindow, + InvalidState, + StoreRejected(RenderedManifestStoreRejectReason), +} + +pub struct HlsManifestRenderer; + +impl HlsManifestRenderer { + pub fn render(session: &HlsSession, rendered_at_ms: u64) -> Result { + for render_gap_segments in 0..=session.render_policy.initial_render_gap_segments { + let Some(window) = select_window(session, render_gap_segments) else { + continue; + }; + let manifest = render_window(session, &window, render_gap_segments, rendered_at_ms)?; + return Ok(manifest); + } + + Err(RenderError::NoRenderableWindow) + } +} + +impl HlsSession { + pub fn render_and_store_manifest(&mut self, rendered_at_ms: u64) -> Result { + let rendered = HlsManifestRenderer::render(self, rendered_at_ms)?; + match self.store_rendered_manifest(rendered.clone()) { + RenderedManifestStoreOutcome::Stored => Ok(rendered), + RenderedManifestStoreOutcome::Rejected(reason) => Err(RenderError::StoreRejected(reason)), + } + } + + pub fn store_rendered_manifest(&mut self, rendered: RenderedManifest) -> RenderedManifestStoreOutcome { + if let Some(previous) = &self.last_rendered_manifest { + if rendered.first_proxy_seq < previous.first_proxy_seq { + return RenderedManifestStoreOutcome::Rejected( + RenderedManifestStoreRejectReason::RegressiveMediaSequence { + previous_first_proxy_seq: previous.first_proxy_seq, + candidate_first_proxy_seq: rendered.first_proxy_seq, + }, + ); + } + } + self.longest_rendered_playlist_duration_ms = + self.longest_rendered_playlist_duration_ms.max(rendered.playlist_duration_ms); + for proxy_seq in &rendered.segment_proxy_seqs { + if let Some(segment) = self.segments.get_mut(proxy_seq) { + segment.last_rendered_at_ms = Some(rendered.rendered_at_ms); + } + } + self.last_rendered_manifest = Some(rendered); + RenderedManifestStoreOutcome::Stored + } +} + +pub fn renderer_candidate_window_proxy_seqs(session: &HlsSession) -> Vec { + for render_gap_segments in 0..=session.render_policy.initial_render_gap_segments { + if let Some(window) = select_window(session, render_gap_segments) { + return window; + } + } + Vec::new() +} + +fn is_renderable(entry: &SegmentEntry, session: &HlsSession) -> bool { + match entry.status { + SegmentCacheStatus::Ready { .. } => {} + SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. } => { + if entry.origin_fetch_ref.is_none() { + return false; + } + } + SegmentCacheStatus::Discovered + | SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::FailedPermanent { .. } + | SegmentCacheStatus::Expired => { + return false; + } + } + let Some(map_ref) = entry.map_ref else { + return true; + }; + session.maps.get(&map_ref).is_some_and(|map| matches!(map.status, MapCacheStatus::Ready { .. })) +} + +fn select_window(session: &HlsSession, render_gap_segments: usize) -> Option> { + let head_seq = session.publishable_origin_head_proxy_seq?; + let tail_seq = session.publishable_origin_tail_proxy_seq?.checked_sub(u64::try_from(render_gap_segments).ok()?)?; + if tail_seq < head_seq { + return None; + } + + let current_origin_window_len = tail_seq.saturating_sub(head_seq).saturating_add(1); + let target_window_len = current_origin_window_len.min(u64::try_from(TARGET_VISIBLE_SEGMENTS).ok()?); + let start_seq = tail_seq + .saturating_add(1) + .saturating_sub(target_window_len) + .max(head_seq); + let mut window = Vec::new(); + let mut not_ready_count = 0_usize; + + for current in start_seq..=tail_seq { + let entry = session.segments.get(¤t)?; + if !is_renderable(entry, session) { + return None; + } + if !matches!(entry.status, SegmentCacheStatus::Ready { .. }) { + not_ready_count = not_ready_count.saturating_add(1); + if not_ready_count > session.render_policy.max_not_ready_render_segments { + return None; + } + } + window.push(current); + } + + if window.len() < MIN_VISIBLE_SEGMENTS { + return None; + } + if session.render_policy.initial_render_gap_segments == 0 && !window_contains_provisioning_segment(session, &window) { + let target_duration_ms = u64::from(resolve_target_duration(session, &window)).saturating_mul(1_000); + if playlist_duration_ms(session, &window) < target_duration_ms.saturating_mul(3) { + return None; + } + } + Some(window) +} + +fn render_window( + session: &HlsSession, + window: &[u64], + render_gap_segments: usize, + rendered_at_ms: u64, +) -> Result { + let Some((&first_proxy_seq, &last_proxy_seq)) = window.first().zip(window.last()) else { + return Err(RenderError::InvalidState); + }; + let playlist_duration_ms = playlist_duration_ms(session, window); + let target_duration = resolve_target_duration(session, window); + let hls_version = resolve_hls_version(session, window); + let discontinuity_sequence = + session.discontinuity_sequence + hidden_discontinuities_before(session, first_proxy_seq); + let mut body = String::new(); + + body.push_str("#EXTM3U\n"); + writeln!(body, "#EXT-X-VERSION:{hls_version}").map_err(|_| RenderError::InvalidState)?; + if session.independent_segments || window_contains_provisioning_segment(session, window) { + body.push_str("#EXT-X-INDEPENDENT-SEGMENTS\n"); + } + writeln!(body, "#EXT-X-TARGETDURATION:{target_duration}").map_err(|_| RenderError::InvalidState)?; + writeln!(body, "#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}").map_err(|_| RenderError::InvalidState)?; + writeln!(body, "#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}") + .map_err(|_| RenderError::InvalidState)?; + + let mut current_map_ref = None; + let contains_provisioning = window_contains_provisioning_segment(session, window); + let mut media_units_rendered = 0_usize; + for proxy_seq in window { + let entry = session.segments.get(proxy_seq).ok_or(RenderError::InvalidState)?; + if contains_provisioning && media_units_rendered == 0 { + append_manifest_block_separator(&mut body); + } + for daterange in &entry.daterange_tags_before { + body.push_str(daterange); + body.push('\n'); + } + if let Some(program_date_time) = &entry.program_date_time { + writeln!(body, "#EXT-X-PROGRAM-DATE-TIME:{program_date_time}") + .map_err(|_| RenderError::InvalidState)?; + } + if entry.discontinuity_before { + if contains_provisioning { + append_manifest_block_separator(&mut body); + } + body.push_str("#EXT-X-DISCONTINUITY\n"); + } + if is_hls_provisioning_gap_segment(entry) { + if contains_provisioning { + append_manifest_block_separator(&mut body); + } + body.push_str("#EXT-X-GAP\n"); + } + if entry.map_ref != current_map_ref { + if let Some(map_ref) = entry.map_ref { + let map = session.maps.get(&map_ref).ok_or(RenderError::InvalidState)?; + writeln!( + body, + "#EXT-X-MAP:URI=\"/hls/shared/live/{}/{}/map/{:06}.{}\"", + session.proxy_session_id.0, HLS_ACCESS_LEASE_ID_PLACEHOLDER, map.proxy_map_id.0, map.proxy_file_ext + ) + .map_err(|_| RenderError::InvalidState)?; + } + current_map_ref = entry.map_ref; + } + writeln!(body, "#EXTINF:{},", format_duration_ms(entry.duration_ms)) + .map_err(|_| RenderError::InvalidState)?; + if is_local_provisioning_segment(entry) { + writeln!( + body, + "/hls/shared/live/{}/{}/{:06}.{}?pseq={}", + proxy_session_id(session), + HLS_ACCESS_LEASE_ID_PLACEHOLDER, + entry.proxy_seq, + entry.proxy_file_ext, + entry.proxy_seq + ) + .map_err(|_| RenderError::InvalidState)?; + } else { + writeln!( + body, + "/hls/shared/live/{}/{}/{:06}.{}", + proxy_session_id(session), + HLS_ACCESS_LEASE_ID_PLACEHOLDER, + entry.proxy_seq, + entry.proxy_file_ext + ) + .map_err(|_| RenderError::InvalidState)?; + } + media_units_rendered = media_units_rendered.saturating_add(1); + } + + Ok(RenderedManifest { + body, + first_proxy_seq, + last_proxy_seq, + playlist_duration_ms, + valid_until_ms: rendered_at_ms.saturating_add(playlist_duration_ms), + render_gap_segments, + rendered_at_ms, + segment_proxy_seqs: window.to_vec(), + }) +} + +fn append_manifest_block_separator(body: &mut String) { + if body.is_empty() || body.ends_with("\n\n") { + return; + } + if !body.ends_with('\n') { + body.push('\n'); + } + body.push('\n'); +} + +fn proxy_session_id(session: &HlsSession) -> &str { + let ProxySessionId(value) = &session.proxy_session_id; + value +} + +fn resolve_hls_version(session: &HlsSession, window: &[u64]) -> u16 { + if window_contains_provisioning_segment(session, window) { + return 7; + } + let needs_map_version = + window.iter().filter_map(|proxy_seq| session.segments.get(proxy_seq)).any(|entry| entry.map_ref.is_some()); + let feature_version = if needs_map_version { 6 } else { 3 }; + session.origin_version.unwrap_or(3).max(feature_version) +} + +fn resolve_target_duration(session: &HlsSession, window: &[u64]) -> u32 { + if window_contains_provisioning_segment(session, window) && !window_contains_origin_segment(session, window) { + return HLS_PROVISIONING_TARGET_DURATION_SECS; + } + session.target_duration.unwrap_or_else(|| { + window + .iter() + .filter_map(|proxy_seq| session.segments.get(proxy_seq)) + .map(|entry| entry.duration_ms.div_ceil(1_000)) + .max() + .unwrap_or(1) + .max(1) + .try_into() + .unwrap_or(u32::MAX) + }) +} + +fn window_contains_provisioning_segment(session: &HlsSession, window: &[u64]) -> bool { + window + .iter() + .filter_map(|proxy_seq| session.segments.get(proxy_seq)) + .any(is_local_provisioning_segment) +} + +fn window_contains_origin_segment(session: &HlsSession, window: &[u64]) -> bool { + window + .iter() + .filter_map(|proxy_seq| session.segments.get(proxy_seq)) + .any(|entry| !is_local_provisioning_segment(entry)) +} + +fn is_local_provisioning_segment(entry: &SegmentEntry) -> bool { + is_hls_provisioning_segment(entry) || is_hls_provisioning_gap_segment(entry) +} + +fn playlist_duration_ms(session: &HlsSession, window: &[u64]) -> u64 { + window.iter().filter_map(|proxy_seq| session.segments.get(proxy_seq)).map(|entry| entry.duration_ms).sum() +} + +fn hidden_discontinuities_before(session: &HlsSession, first_proxy_seq: u64) -> u64 { + session + .segments + .range(..first_proxy_seq) + .filter(|(_, entry)| entry.discontinuity_before) + .count() + .try_into() + .unwrap_or(u64::MAX) +} + +fn format_duration_ms(duration_ms: u64) -> String { format!("{}.{:03}", duration_ms / 1_000, duration_ms % 1_000) } + +#[cfg(test)] +mod tests { + use shared::model::HlsStripMode; + use super::{ + HlsManifestRenderer, RenderError, RenderPolicy, RenderedManifest, RenderedManifestStoreOutcome, + RenderedManifestStoreRejectReason, + }; + use crate::{ + api::model::{HlsSession, HlsSessionKey, MapCacheStatus, SegmentCacheStatus, SegmentFetchPriority}, + model::StripConfig, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + }; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn session() -> HlsSession { HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0) } + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + fn six_segment_manifest() -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:120\n#EXTINF:4.0,\norigin-name-120.ts\n#EXTINF:4.0,\norigin-name-121.ts\n#EXTINF:4.0,\norigin-name-122.ts\n#EXTINF:4.0,\norigin-name-123.ts\n#EXTINF:4.0,\norigin-name-124.ts\n#EXTINF:4.0,\norigin-name-125.ts\n", + ) + } + + fn mark_all_segments_ready(session: &mut HlsSession) { + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1024, ready_at_ms: 10 }; + } + } + + fn rendered_manifest(first_proxy_seq: u64, last_proxy_seq: u64) -> RenderedManifest { + RenderedManifest { + body: format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}\n"), + first_proxy_seq, + last_proxy_seq, + playlist_duration_ms: 4_000, + valid_until_ms: 4_000, + render_gap_segments: 0, + rendered_at_ms: first_proxy_seq, + segment_proxy_seqs: (first_proxy_seq..=last_proxy_seq).collect(), + } + } + + #[test] + fn rendering_an_empty_internal_window_does_not_panic() { + let session = session(); + + let rendered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + super::render_window(&session, &[], 0, 0) + })); + + assert!(rendered.is_ok()); + } + + #[test] + fn renderer_does_not_emit_origin_uri_or_file_name() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(!rendered.body.contains("origin.example.com")); + assert!(!rendered.body.contains("origin-name-120.ts")); + assert!(rendered.body.contains("/hls/shared/live/")); + } + + #[test] + fn renderer_emits_six_digit_proxy_sequence_urls() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(rendered.body.contains("/000000.ts")); + } + + #[test] + fn media_sequence_is_first_visible_proxy_sequence() { + let mut session = session(); + session.render_policy = RenderPolicy::new(1); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert_eq!(rendered.first_proxy_seq, 0); + assert!(rendered.body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert_eq!(rendered.render_gap_segments, 0); + } + + #[test] + fn discontinuity_and_sequence_are_rendered_for_visible_window() { + let mut session = session(); + let first = normal_manifest( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:3\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n#EXTINF:4.0,\n7.ts\n", + ); + session.apply_origin_manifest(&first).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert_eq!(rendered.first_proxy_seq, 1); + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:4\n")); + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,\n/hls/shared/live/")); + } + + #[test] + fn provisioning_handoff_discontinuity_is_rendered_for_first_origin_segment() { + let mut session = session(); + session.mark_pending_handoff_discontinuity(0); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:7\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\n10.ts\n#EXTINF:4.0,\n11.ts\n#EXTINF:4.0,\n12.ts\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:7\n")); + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,\n/hls/shared/live/")); + assert_eq!(rendered.body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert_eq!(session.pending_handoff_discontinuity_sequence, None); + } + + #[test] + fn provisioning_handoff_does_not_duplicate_origin_first_segment_discontinuity() { + let mut session = session(); + session.mark_pending_handoff_discontinuity(0); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:7\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\n10.ts\n#EXTINF:4.0,\n11.ts\n#EXTINF:4.0,\n12.ts\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:7\n")); + assert_eq!(rendered.body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + } + + #[test] + fn byterange_is_not_rendered() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-BYTERANGE:500@1000\n#EXTINF:4.0,\nbig.m4s\n#EXT-X-BYTERANGE:500\n#EXTINF:4.0,\nbig.m4s\n#EXT-X-BYTERANGE:500\n#EXTINF:4.0,\nbig.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(!rendered.body.contains("#EXT-X-BYTERANGE")); + } + + #[test] + fn queued_and_fetching_segments_with_fetch_ref_are_renderable_with_limit() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.segments.get_mut(&0).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 100, ready_at_ms: 1 }; + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 100, ready_at_ms: 1 }; + session.segments.get_mut(&2).expect("segment").status = + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, queued_at_ms: 1 }; + session.segments.get_mut(&3).expect("segment").status = + SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::RenderWindow, started_at_ms: 1 }; + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert_eq!(rendered.segment_proxy_seqs, vec![0, 1, 2, 3]); + } + + #[test] + fn max_not_ready_render_segments_blocks_too_many_queued_segments() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + for segment in session.segments.values_mut() { + segment.status = + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, queued_at_ms: 1 }; + } + + assert_eq!(HlsManifestRenderer::render(&session, 10), Err(RenderError::NoRenderableWindow)); + } + + #[test] + fn map_not_ready_prevents_rendering_affected_segments() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + assert_eq!(HlsManifestRenderer::render(&session, 10), Err(RenderError::NoRenderableWindow)); + } + + #[test] + fn current_origin_manifest_head_blocks_suffix_fallback() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + session.segments.get_mut(&0).expect("segment 0").status = SegmentCacheStatus::Discovered; + session.segments.get_mut(&1).expect("segment 1").status = SegmentCacheStatus::Discovered; + + assert_eq!(HlsManifestRenderer::render(&session, 10), Err(RenderError::NoRenderableWindow)); + } + + #[test] + fn map_ready_renders_only_proxy_map_uri() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"origin-init-name.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + for map in session.maps.values_mut() { + map.status = MapCacheStatus::Ready { content_length: 128, ready_at_ms: 10 }; + } + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert!(rendered.body.contains("/hls/shared/live/")); + assert!(rendered.body.contains("/map/000000.mp4")); + assert!(!rendered.body.contains("origin-init-name.mp4")); + } + + #[test] + fn map_tag_is_rendered_once_for_unchanged_map_ref() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n#EXTINF:4.0,\n4.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + for map in session.maps.values_mut() { + map.status = MapCacheStatus::Ready { content_length: 128, ready_at_ms: 10 }; + } + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert_eq!(rendered.body.matches("#EXT-X-MAP").count(), 1); + } + + #[test] + fn map_tag_is_rendered_again_when_map_ref_changes() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init-a.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n#EXT-X-MAP:URI=\"init-b.mp4\"\n#EXTINF:4.0,\n4.m4s\n#EXTINF:4.0,\n5.m4s\n#EXTINF:4.0,\n6.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + for map in session.maps.values_mut() { + map.status = MapCacheStatus::Ready { content_length: 128, ready_at_ms: 10 }; + } + + let rendered = HlsManifestRenderer::render(&session, 10).expect("manifest should render"); + + assert_eq!(rendered.body.matches("#EXT-X-MAP").count(), 2); + assert!(rendered.body.contains("/map/000000.mp4")); + assert!(rendered.body.contains("/map/000001.mp4")); + } + + #[test] + fn renderer_gap_is_relative_to_publishable_tail() { + let mut session = session(); + session.render_policy = RenderPolicy::new(3); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n5.m4s\n#EXTINF:4.0,\n6.m4s\n#EXTINF:4.0,\n7.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + let rendered = HlsManifestRenderer::render(&session, 10).expect("older valid window should render"); + + assert_eq!(session.publishable_origin_tail_proxy_seq, Some(6)); + assert_eq!(rendered.last_proxy_seq, 3); + assert_eq!(rendered.render_gap_segments, 3); + assert_eq!(rendered.segment_proxy_seqs, vec![0, 1, 2, 3]); + } + + #[test] + fn store_rendered_manifest_rejects_regressive_media_sequence() { + let mut session = session(); + let previous = rendered_manifest(2, 7); + let candidate = rendered_manifest(0, 5); + + assert_eq!(session.store_rendered_manifest(previous.clone()), RenderedManifestStoreOutcome::Stored); + assert_eq!( + session.store_rendered_manifest(candidate), + RenderedManifestStoreOutcome::Rejected(RenderedManifestStoreRejectReason::RegressiveMediaSequence { + previous_first_proxy_seq: 2, + candidate_first_proxy_seq: 0, + }) + ); + assert_eq!(session.last_rendered_manifest, Some(previous)); + } + + #[test] + fn render_and_store_manifest_returns_error_on_store_rejection() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + session.last_rendered_manifest = Some(rendered_manifest(2, 7)); + + assert_eq!( + session.render_and_store_manifest(10), + Err(RenderError::StoreRejected(RenderedManifestStoreRejectReason::RegressiveMediaSequence { + previous_first_proxy_seq: 2, + candidate_first_proxy_seq: 0, + })) + ); + assert_eq!(session.last_rendered_manifest, Some(rendered_manifest(2, 7))); + } + + #[test] + fn store_rendered_manifest_accepts_same_or_forward_media_sequence() { + let mut session = session(); + let first = rendered_manifest(2, 7); + let same = rendered_manifest(2, 8); + let forward = rendered_manifest(3, 9); + + assert_eq!(session.store_rendered_manifest(first), RenderedManifestStoreOutcome::Stored); + assert_eq!(session.store_rendered_manifest(same.clone()), RenderedManifestStoreOutcome::Stored); + assert_eq!(session.last_rendered_manifest, Some(same)); + assert_eq!(session.store_rendered_manifest(forward.clone()), RenderedManifestStoreOutcome::Stored); + assert_eq!(session.last_rendered_manifest, Some(forward)); + } + + #[test] + fn renderer_does_not_exceed_initial_render_gap_to_find_valid_window() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + let first = session.render_and_store_manifest(10).expect("initial manifest should render"); + + session.render_policy = RenderPolicy::new(2); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:126\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n126.m4s\n#EXTINF:4.0,\n127.m4s\n#EXTINF:4.0,\n128.m4s\n", + ); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + mark_all_segments_ready(&mut session); + + assert_eq!(HlsManifestRenderer::render(&session, 20), Err(RenderError::NoRenderableWindow)); + assert_eq!(session.render_and_store_manifest(20), Err(RenderError::NoRenderableWindow)); + assert_eq!(session.last_rendered_manifest, Some(first)); + } + + #[test] + fn invalid_render_does_not_replace_last_rendered_manifest() { + let mut session = session(); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); + mark_all_segments_ready(&mut session); + let first = session.render_and_store_manifest(10).expect("manifest should render"); + session.segments.clear(); + + assert_eq!(session.render_and_store_manifest(20), Err(RenderError::NoRenderableWindow)); + assert_eq!(session.last_rendered_manifest, Some(first)); + } + + #[test] + fn render_policy_from_seconds_counts_tail_durations() { + let policy = RenderPolicy::from_strip_config( + &StripConfig { mode: HlsStripMode::Seconds, value: 9 }, + &[4_000, 4_000, 4_000, 4_000], + ); + + assert_eq!(policy.initial_render_gap_segments, 3); + } +} diff --git a/backend/src/api/model/hls_cache/resource_fetch.rs b/backend/src/api/model/hls_cache/resource_fetch.rs new file mode 100644 index 000000000..6083a2e5b --- /dev/null +++ b/backend/src/api/model/hls_cache/resource_fetch.rs @@ -0,0 +1,723 @@ +use super::{ + append_hls_provider_session_headers, force_identity_without_range, hls_object_body_deadline, safe_origin_log_value, + safe_proxy_session_id, scrub_hls_origin_headers, HlsBoundAccountAcquireErrorKind, ProxySessionId, + SegmentFetchPolicy, +}; +use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; +use futures::{future::BoxFuture, FutureExt}; +use log::{debug, warn}; +use reqwest::Client; +use shared::utils::sanitize_sensitive_info; +use std::{ + io, + time::{Duration, Instant}, +}; +use url::Url; + +const MAX_MANUAL_REDIRECTS: usize = 10; +const STORAGE_FULL_RAW_OS_ERRORS: &[i32] = &[28, 112, 122]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceStatusClass { + Success, + Retryable, + Permanent, + NonRetryable, +} + +pub fn classify_hls_resource_status(status: StatusCode) -> HlsResourceStatusClass { + if status.is_success() { + return HlsResourceStatusClass::Success; + } + if status.is_server_error() + || matches!( + status, + StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_EARLY + | StatusCode::TOO_MANY_REQUESTS + ) + { + return HlsResourceStatusClass::Retryable; + } + if matches!( + status, + StatusCode::BAD_REQUEST + | StatusCode::UNAUTHORIZED + | StatusCode::FORBIDDEN + | StatusCode::NOT_FOUND + | StatusCode::GONE + ) { + return HlsResourceStatusClass::Permanent; + } + HlsResourceStatusClass::NonRetryable +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceFetchKind { + Segment, + Map, + Key, + Other, +} + +impl HlsResourceFetchKind { + pub fn label(self) -> &'static str { + match self { + Self::Segment => "Segment", + Self::Map => "Map", + Self::Key => "Key", + Self::Other => "Resource", + } + } + + pub fn operation(self) -> &'static str { + match self { + Self::Segment => "segment fetch", + Self::Map => "map fetch", + Self::Key => "key fetch", + Self::Other => "resource fetch", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceFetchSource { + Normal, + Transient, +} + +impl HlsResourceFetchSource { + pub fn as_log_value(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::Transient => "transient", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsOriginByteRangeExpectation { + FullObject, + PartialContent, + AnySuccess, +} + +#[derive(Debug, Clone)] +pub struct HlsOriginResourceFetchTarget { + pub kind: HlsResourceFetchKind, + pub source: HlsResourceFetchSource, + pub object_id: String, + pub origin_url: String, + pub headers: HeaderMap, + pub byte_range_expectation: HlsOriginByteRangeExpectation, +} + +impl HlsOriginResourceFetchTarget { + pub fn log_context(&self) -> HlsResourceFetchLogContext<'_> { + HlsResourceFetchLogContext { + kind: self.kind, + source: self.source, + object_id: &self.object_id, + origin_url: Some(&self.origin_url), + } + } +} + +#[derive(Clone)] +pub struct HlsOriginResourceClients { + pub client: Client, + pub no_redirect_client: Client, + pub use_manual_redirects: bool, +} + +#[derive(Debug)] +pub enum HlsOriginResourceFetchError { + PermanentStatus(StatusCode), + RetryableStatus(StatusCode), + NonRetryableStatus(StatusCode), + Transport(String), + Redirect, + Timeout, + InvalidOriginUrl, + InvalidByteRange, + UnexpectedByteRangeStatus, + CacheCommit(HlsCacheCommitFailure), + ProviderUnavailable(HlsBoundAccountAcquireErrorKind), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsCacheCommitFailure { + io_kind: String, + raw_os_error: Option, + storage_full: bool, + message: String, +} + +impl HlsCacheCommitFailure { + pub fn from_io_error(err: &io::Error) -> Self { + let raw_os_error = err.raw_os_error(); + let io_kind = format!("{:?}", err.kind()); + Self { + io_kind, + raw_os_error, + storage_full: matches!(err.kind(), io::ErrorKind::StorageFull) + || raw_os_error.is_some_and(|code| STORAGE_FULL_RAW_OS_ERRORS.contains(&code)), + message: sanitize_sensitive_info(&err.to_string()).to_string(), + } + } + + fn label(&self) -> String { + let raw_os_error = self + .raw_os_error + .map_or_else(|| "none".to_string(), |code| code.to_string()); + format!( + "CacheCommitError(io_kind={} raw_os_error={} storage_full={} message=\"{}\")", + self.io_kind, raw_os_error, self.storage_full, self.message + ) + } + + pub fn storage_full(&self) -> bool { self.storage_full } +} + +impl HlsOriginResourceFetchError { + pub fn cache_commit(err: &io::Error) -> Self { Self::CacheCommit(HlsCacheCommitFailure::from_io_error(err)) } + + pub fn retryable_failure(&self) -> bool { + matches!( + self, + Self::RetryableStatus(_) | Self::Transport(_) | Self::Redirect | Self::Timeout + ) + || matches!(self, Self::CacheCommit(failure) if !failure.storage_full()) + || matches!(self, Self::ProviderUnavailable(kind) if kind.is_retryable_resource_failure()) + } + + pub fn permanent_status(&self) -> Option { + match self { + Self::PermanentStatus(status) | Self::NonRetryableStatus(status) => Some(*status), + _ => None, + } + } + + pub fn object_failure_is_permanent(&self) -> bool { + matches!( + self, + Self::PermanentStatus(_) + | Self::NonRetryableStatus(_) + | Self::InvalidOriginUrl + | Self::InvalidByteRange + | Self::UnexpectedByteRangeStatus + ) + } + + fn log_status(&self) -> HlsResourceFetchLogStatus { + match self { + Self::PermanentStatus(status) | Self::RetryableStatus(status) | Self::NonRetryableStatus(status) => { + HlsResourceFetchLogStatus::Http(*status) + } + Self::Transport(_) | Self::InvalidOriginUrl | Self::InvalidByteRange | Self::UnexpectedByteRangeStatus => { + HlsResourceFetchLogStatus::TransportError + } + Self::Redirect => HlsResourceFetchLogStatus::RedirectError, + Self::Timeout => HlsResourceFetchLogStatus::Timeout, + Self::CacheCommit(failure) => HlsResourceFetchLogStatus::CacheCommitError(failure.clone()), + Self::ProviderUnavailable(kind) => HlsResourceFetchLogStatus::ProviderUnavailable(*kind), + } + } + + fn aborts_without_retry(&self) -> bool { + match self { + Self::PermanentStatus(_) + | Self::NonRetryableStatus(_) + | Self::InvalidOriginUrl + | Self::InvalidByteRange + | Self::UnexpectedByteRangeStatus => true, + Self::ProviderUnavailable(kind) => !kind.is_retryable_resource_failure(), + Self::RetryableStatus(_) + | Self::Transport(_) + | Self::Redirect + | Self::Timeout + | Self::CacheCommit(_) => false, + } + } +} + +pub type HlsOriginResourceCommitFuture = BoxFuture<'static, Result>; +pub type HlsOriginResourceAttemptPrepareFuture = BoxFuture<'static, Result>; +pub type HlsOriginResourceAttemptCleanupFuture = BoxFuture<'static, ()>; + +pub async fn run_hls_origin_resource_retry_loop( + target: HlsOriginResourceFetchTarget, + clients: HlsOriginResourceClients, + policy: &SegmentFetchPolicy, + session_log_id: &str, + mut commit: F, +) -> Result +where + T: Send + 'static, + F: FnMut(reqwest::Response, HlsResourceFetchAttempt) -> HlsOriginResourceCommitFuture, +{ + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + clients, + policy, + session_log_id, + |_| async { Ok(()) }.boxed(), + |()| async {}.boxed(), + move |response, attempt, ()| commit(response, attempt), + ) + .await +} + +/// Runs the shared HLS origin-resource retry policy with an optional per-attempt guard. +/// +/// The prepare callback runs after the attempt log and before the HTTP request. If the HTTP +/// request fails before the commit callback takes ownership of the guard, the cleanup callback is +/// invoked by the runner. Once the commit callback receives a guard, it owns its cleanup path. This +/// lets direct passthrough callers return a guard with the response body while cache-commit callers +/// release it after the commit finishes. +pub async fn run_hls_origin_resource_retry_loop_with_attempt_prepare( + target: HlsOriginResourceFetchTarget, + clients: HlsOriginResourceClients, + policy: &SegmentFetchPolicy, + session_log_id: &str, + mut prepare_attempt: P, + mut cleanup_attempt: C, + mut commit: F, +) -> Result +where + T: Send + 'static, + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> HlsOriginResourceAttemptPrepareFuture, + C: FnMut(G) -> HlsOriginResourceAttemptCleanupFuture, + F: FnMut(reqwest::Response, HlsResourceFetchAttempt, G) -> HlsOriginResourceCommitFuture, +{ + let attempts = policy.retry_delays_ms.len(); + for attempt_index in 0..attempts { + let attempt = HlsResourceFetchAttempt { attempt_index, attempts }; + let delay_ms = policy.retry_delay_ms(attempt_index); + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + log_hls_resource_attempt_started(session_log_id, target.log_context(), attempt); + let attempt_started_at = Instant::now(); + let result = match prepare_attempt(attempt).await { + Ok(guard) => match fetch_hls_origin_resource_response(&target, &clients).await { + Ok(response) => commit(response, attempt, guard).await, + Err(err) => { + cleanup_attempt(guard).await; + Err(err) + } + }, + Err(err) => Err(err), + }; + + match result { + Ok(output) => { + log_hls_resource_attempt_succeeded(session_log_id, target.log_context(), attempt_started_at.elapsed()); + return Ok(output); + } + Err(err) if err.aborts_without_retry() || attempt_index + 1 == attempts => { + if matches!(err, HlsOriginResourceFetchError::Timeout) { + log_hls_resource_timeout( + session_log_id, + target.log_context(), + attempt, + hls_object_body_deadline(policy.origin_segment_timeout_ms).as_millis(), + ); + } + log_hls_resource_fetch_failed(session_log_id, target.log_context(), attempt, err.log_status()); + return Err(err); + } + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::Timeout) { + log_hls_resource_timeout( + session_log_id, + target.log_context(), + attempt, + hls_object_body_deadline(policy.origin_segment_timeout_ms).as_millis(), + ); + } + log_hls_resource_retry_scheduled( + session_log_id, + target.log_context(), + attempt, + err.log_status(), + policy.retry_delays_ms.get(attempt_index + 1).copied().unwrap_or_default(), + ); + } + } + } + + Err(HlsOriginResourceFetchError::Timeout) +} + +pub async fn fetch_hls_origin_resource_response( + target: &HlsOriginResourceFetchTarget, + clients: &HlsOriginResourceClients, +) -> Result { + let url = Url::parse(&target.origin_url).map_err(|_| HlsOriginResourceFetchError::InvalidOriginUrl)?; + let response = if clients.use_manual_redirects { + fetch_hls_origin_resource_with_manual_redirects(&url, target.headers.clone(), &clients.no_redirect_client) + .await? + } else { + clients + .client + .get(url) + .headers(target.headers.clone()) + .send() + .await + .map_err(|err| { + HlsOriginResourceFetchError::Transport( + sanitize_sensitive_info(err.to_string().as_str()).to_string(), + ) + })? + }; + classify_hls_origin_resource_response(response, target.byte_range_expectation) +} + +async fn fetch_hls_origin_resource_with_manual_redirects( + entry_url: &Url, + headers: HeaderMap, + client: &Client, +) -> Result { + let mut current_url = entry_url.clone(); + let mut current_headers = headers; + let mut remaining_redirects = MAX_MANUAL_REDIRECTS; + + loop { + let response = client + .get(current_url.clone()) + .headers(current_headers.clone()) + .send() + .await + .map_err(|err| { + HlsOriginResourceFetchError::Transport( + sanitize_sensitive_info(err.to_string().as_str()).to_string(), + ) + })?; + if !response.status().is_redirection() { + return Ok(response); + } + if remaining_redirects == 0 { + return Err(HlsOriginResourceFetchError::Redirect); + } + let response_url = response.url().clone(); + let location = response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or(HlsOriginResourceFetchError::Redirect)?; + let next_url = response_url + .join(location) + .or_else(|_| Url::parse(location)) + .map_err(|_| HlsOriginResourceFetchError::Redirect)?; + if !same_origin(&response_url, &next_url) { + strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); + } + current_url = next_url; + remaining_redirects = remaining_redirects.saturating_sub(1); + } +} + +fn classify_hls_origin_resource_response( + response: reqwest::Response, + byte_range_expectation: HlsOriginByteRangeExpectation, +) -> Result { + let status = response.status(); + match classify_hls_resource_status(status) { + HlsResourceStatusClass::Success => { + match byte_range_expectation { + HlsOriginByteRangeExpectation::FullObject if status == StatusCode::PARTIAL_CONTENT => { + return Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus); + } + HlsOriginByteRangeExpectation::PartialContent if status != StatusCode::PARTIAL_CONTENT => { + return Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus); + } + HlsOriginByteRangeExpectation::FullObject + | HlsOriginByteRangeExpectation::PartialContent + | HlsOriginByteRangeExpectation::AnySuccess => {} + } + Ok(response) + } + HlsResourceStatusClass::Retryable => Err(HlsOriginResourceFetchError::RetryableStatus(status)), + HlsResourceStatusClass::Permanent => Err(HlsOriginResourceFetchError::PermanentStatus(status)), + HlsResourceStatusClass::NonRetryable => Err(HlsOriginResourceFetchError::NonRetryableStatus(status)), + } +} + +pub fn build_hls_origin_resource_headers( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + byte_range: Option, +) -> Result { + let mut headers = source_headers.clone(); + scrub_hls_origin_headers(&mut headers, None); + force_identity_without_range(&mut headers); + append_hls_provider_session_headers(&mut headers, provider_session_headers); + if let Some(byte_range) = byte_range { + let end = byte_range + .offset + .checked_add(byte_range.length) + .and_then(|end_exclusive| end_exclusive.checked_sub(1)) + .ok_or(HlsOriginResourceFetchError::InvalidByteRange)?; + let range_value = format!("bytes={}-{}", byte_range.offset, end); + let range_value = HeaderValue::from_str(&range_value) + .map_err(|_| HlsOriginResourceFetchError::InvalidByteRange)?; + headers.insert(header::RANGE, range_value); + } + Ok(headers) +} + +pub fn build_hls_origin_resource_headers_with_client_range( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + client_range: Option, +) -> HeaderMap { + let mut headers = source_headers.clone(); + scrub_hls_origin_headers(&mut headers, None); + headers.remove(header::RANGE); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("identity")); + append_hls_provider_session_headers(&mut headers, provider_session_headers); + if let Some(range) = client_range { + headers.insert(header::RANGE, range); + } + headers +} + +fn same_origin(lhs: &Url, rhs: &Url) -> bool { + lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) + && lhs.host_str() == rhs.host_str() + && lhs.port_or_known_default() == rhs.port_or_known_default() +} + +fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HeaderMap) { + scrub_hls_origin_headers(headers, None); +} + +#[derive(Debug, Clone, Copy)] +pub struct HlsResourceFetchLogContext<'a> { + pub kind: HlsResourceFetchKind, + pub source: HlsResourceFetchSource, + pub object_id: &'a str, + pub origin_url: Option<&'a str>, +} + +#[derive(Debug, Clone)] +pub enum HlsResourceFetchLogStatus { + Http(StatusCode), + Timeout, + TransportError, + RedirectError, + CacheCommitError(HlsCacheCommitFailure), + ProviderUnavailable(HlsBoundAccountAcquireErrorKind), +} + +impl HlsResourceFetchLogStatus { + pub fn label(self) -> String { + match self { + Self::Http(status) => { + let reason = status.canonical_reason().unwrap_or("Unknown"); + format!("{} {reason}", status.as_u16()) + } + Self::Timeout => "Timeout".to_string(), + Self::TransportError => "TransportError".to_string(), + Self::RedirectError => "RedirectError".to_string(), + Self::CacheCommitError(failure) => failure.label(), + Self::ProviderUnavailable(kind) => format!("ProviderUnavailable({})", kind.as_log_label()), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct HlsResourceFetchAttempt { + pub attempt_index: usize, + pub attempts: usize, +} + +fn safe_resource_fetch_session_id(session: &str) -> String { + safe_proxy_session_id(&ProxySessionId(session.to_string())) +} + +pub fn log_hls_resource_attempt_started( + session: &str, + context: HlsResourceFetchLogContext<'_>, + attempt: HlsResourceFetchAttempt, +) { + if let Some(origin_url) = context.origin_url { + debug!( + "{} '{}' attempting URL attempt {} of {}: session={} source={} {}", + context.kind.label(), + context.object_id, + attempt.attempt_index, + attempt.attempts, + safe_resource_fetch_session_id(session), + context.source.as_log_value(), + safe_origin_log_value(origin_url) + ); + } else { + debug!( + "{} '{}' attempting URL attempt {} of {}: session={} source={}", + context.kind.label(), + context.object_id, + attempt.attempt_index, + attempt.attempts, + safe_resource_fetch_session_id(session), + context.source.as_log_value() + ); + } +} + +pub fn log_hls_resource_attempt_succeeded( + session: &str, + context: HlsResourceFetchLogContext<'_>, + elapsed: Duration, +) { + debug!( + "{} '{}' success: session={} source={} {} took {:.3}s", + context.kind.label(), + context.object_id, + safe_resource_fetch_session_id(session), + context.source.as_log_value(), + context.kind.operation(), + elapsed.as_secs_f64() + ); +} + +pub fn log_hls_resource_retry_scheduled( + session: &str, + context: HlsResourceFetchLogContext<'_>, + attempt: HlsResourceFetchAttempt, + status: HlsResourceFetchLogStatus, + next_delay_ms: u64, +) { + warn!( + "{} '{}' retry scheduled: session={} source={} status {} attempt {} of {} next_delay_ms={}", + context.kind.label(), + context.object_id, + safe_resource_fetch_session_id(session), + context.source.as_log_value(), + status.label(), + attempt.attempt_index, + attempt.attempts, + next_delay_ms + ); +} + +pub fn log_hls_resource_fetch_failed( + session: &str, + context: HlsResourceFetchLogContext<'_>, + attempt: HlsResourceFetchAttempt, + status: HlsResourceFetchLogStatus, +) { + warn!( + "{} '{}' failed: session={} source={} status {} attempt {} of {}", + context.kind.label(), + context.object_id, + safe_resource_fetch_session_id(session), + context.source.as_log_value(), + status.label(), + attempt.attempt_index, + attempt.attempts + ); +} + +pub fn log_hls_resource_timeout( + session: &str, + context: HlsResourceFetchLogContext<'_>, + attempt: HlsResourceFetchAttempt, + deadline_ms: u128, +) { + warn!( + "HLS origin object fetch timed out: session={} source={} kind={} object={} attempt={} of {} deadline_ms={}", + safe_resource_fetch_session_id(session), + context.source.as_log_value(), + context.kind.label(), + context.object_id, + attempt.attempt_index, + attempt.attempts, + deadline_ms + ); +} + +pub fn retry_after_secs_from_ms(retry_after_ms: u64) -> u64 { + retry_after_ms.saturating_add(999).saturating_div(1_000).max(1) +} + +#[cfg(test)] +mod tests { + use super::{ + build_hls_origin_resource_headers_with_client_range, HlsCacheCommitFailure, HlsOriginResourceFetchError, + HlsResourceFetchLogStatus, + }; + use crate::api::model::HlsBoundAccountAcquireErrorKind; + use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; + use std::io; + + #[test] + fn formats_http_fetch_log_status_with_reason() { + assert_eq!( + HlsResourceFetchLogStatus::Http(StatusCode::INTERNAL_SERVER_ERROR).label(), + "500 Internal Server Error" + ); + } + + #[test] + fn formats_non_http_fetch_log_status() { + assert_eq!(HlsResourceFetchLogStatus::Timeout.label(), "Timeout"); + assert_eq!(HlsResourceFetchLogStatus::TransportError.label(), "TransportError"); + assert_eq!(HlsResourceFetchLogStatus::RedirectError.label(), "RedirectError"); + } + + #[test] + fn cache_commit_failure_marks_enospc_as_storage_full() { + let failure = HlsCacheCommitFailure::from_io_error(&io::Error::from_raw_os_error(28)); + + assert!(failure.storage_full()); + assert!(HlsResourceFetchLogStatus::CacheCommitError(failure) + .label() + .contains("storage_full=true")); + } + + #[test] + fn storage_full_cache_commit_failure_is_not_retryable() { + let err = HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28)); + + assert!(!err.retryable_failure()); + } + + #[test] + fn provider_wait_timeout_is_retryable_resource_failure() { + let err = HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::WaitTimedOut); + + assert!(err.retryable_failure()); + assert!(!err.aborts_without_retry()); + } + + #[test] + fn missing_provider_account_aborts_resource_retry() { + let err = HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Missing); + + assert!(!err.retryable_failure()); + assert!(err.aborts_without_retry()); + } + + #[test] + fn origin_resource_headers_drop_client_cookie_and_append_provider_cookie() { + let mut client_headers = HeaderMap::new(); + client_headers.insert(header::COOKIE, HeaderValue::from_static("client=secret")); + client_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + + let mut provider_headers = HeaderMap::new(); + provider_headers.insert(header::COOKIE, HeaderValue::from_static("sid=provider")); + + let headers = + build_hls_origin_resource_headers_with_client_range(&client_headers, &provider_headers, None); + + assert_eq!(headers.get(header::COOKIE).expect("cookie"), "sid=provider"); + assert!(!headers.contains_key(header::RANGE)); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); + } +} diff --git a/backend/src/api/model/hls_cache/response.rs b/backend/src/api/model/hls_cache/response.rs new file mode 100644 index 000000000..001e83ef1 --- /dev/null +++ b/backend/src/api/model/hls_cache/response.rs @@ -0,0 +1,1084 @@ +#![allow(clippy::large_futures, clippy::large_enum_variant, clippy::too_many_lines)] + +use super::{ + hls_client_body_send_deadline, refresh_hls_client_body_send_deadline, safe_hls_access_lease_id, + safe_proxy_session_id, CacheAccessState, + HlsAccessLeaseId, HlsCacheMetrics, HlsMapFile, HlsProxyManager, HlsRepairRenderedObjectId, HlsSegmentCache, + HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, + HlsSessionHandle, MapCacheKey, MapCacheStatus, ProxyMapId, SegmentCacheKey, SegmentCacheStatus, + TransientObjectCacheKey, TransientResourceFile, TransientResourceKind, +}; +use crate::api::{api_utils::mark_response_as_uncompressed, model::StreamMeterHandle}; +use arc_swap::ArcSwapOption; +use axum::{ + body::Body, + http::{header, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use futures::Stream; +use log::debug; +use std::{ + io, + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::{Duration, Instant}, +}; +use tokio::io::AsyncReadExt; +use tokio::time::{sleep, Sleep}; +use tokio_util::io::ReaderStream; + +const ACCEPT_RANGES_VALUE: &str = "bytes"; +const NOT_READY_RETRY_AFTER_MS: u64 = 1_000; +const BODY_READER_WAIT_LOG_THRESHOLD_MS: u128 = 10; +static NEXT_HLS_BODY_LOG_ID: AtomicU64 = AtomicU64::new(1); + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RangeDecision { + Full, + Partial { start: u64, end: u64, length: u64 }, + Unsatisfiable, +} + +#[derive(Clone)] +struct CacheObject { + key: K, + access: Arc, + content_type: String, + log_context: CacheObjectLogContext, + repair_context: Option, +} + +#[derive(Clone)] +struct CacheObjectLogContext { + lease: String, + session: String, + resource_id: String, + object_kind: &'static str, + body_source: &'static str, +} + +#[derive(Clone)] +struct CacheBodyLogContext { + session: String, + resource_id: String, + object_kind: &'static str, + source: &'static str, + content_length: u64, +} + +enum CacheObjectLookup { + Ready(CacheObject), + Failure(HlsResourceServeFailure), +} + +/// Result of a Shared-HLS cache object serve decision. +pub enum HlsResourceServeOutcome { + Ready(Response), + Failure(HlsResourceServeFailure), +} + +/// Typed Shared-HLS resource failure before endpoint-level custom-response mapping. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceServeFailure { + TemporaryUnavailable { retry_after_ms: u64 }, + Missing, + Expired, + PermanentFailed { status: Option }, +} + +pub struct HlsCacheResponseContext { + pub hls_access_lease_id: HlsAccessLeaseId, + pub cache_duration_seconds: u64, + pub metrics: Arc, + pub segment_repair: Arc, + pub qos_meter: Arc>, + pub media_activity_marker: Option, + pub now_ms: u64, +} + +impl HlsCacheResponseContext { + pub fn new( + hls_access_lease_id: HlsAccessLeaseId, + cache_duration_seconds: u64, + metrics: Arc, + segment_repair: Arc, + qos_meter: Option>, + media_activity_marker: Option, + now_ms: u64, + ) -> Self { + Self { + hls_access_lease_id, + cache_duration_seconds, + metrics, + segment_repair, + qos_meter: Arc::new(ArcSwapOption::from(qos_meter)), + media_activity_marker, + now_ms, + } + } + + pub fn set_qos_meter(&self, qos_meter: Option>) { + self.qos_meter.store(qos_meter); + } +} + +#[derive(Clone)] +pub struct HlsMediaActivityMarker { + manager: Arc, + session: HlsSessionHandle, +} + +impl HlsMediaActivityMarker { + pub fn new(manager: Arc, session: HlsSessionHandle) -> Self { Self { manager, session } } + + pub async fn mark_at(&self, now_ms: u64) { + self.manager.mark_authorized_media_access_for_session(&self.session, now_ms).await; + } + + pub fn spawn_mark_at(&self, now_ms: u64) { + let marker = self.clone(); + tokio::spawn(async move { + marker.mark_at(now_ms).await; + }); + } + + pub fn spawn_mark_now(&self) { self.spawn_mark_at(current_time_millis()); } +} + +struct CacheObjectServeContext { + cache_duration_seconds: u64, + metrics: Option>, + segment_repair: Arc, + qos_meter: Arc>, + media_activity_marker: Option, + now_ms: u64, +} + +impl CacheObjectServeContext { + fn from_response_context(context: &HlsCacheResponseContext) -> Self { + Self { + cache_duration_seconds: context.cache_duration_seconds, + metrics: Some(Arc::clone(&context.metrics)), + segment_repair: Arc::clone(&context.segment_repair), + qos_meter: Arc::clone(&context.qos_meter), + media_activity_marker: context.media_activity_marker.clone(), + now_ms: context.now_ms, + } + } +} + +/// Serves a committed Ready segment from the HLS cache. +pub async fn serve_hls_segment_cache_response( + segment_cache: Arc, + session: HlsSessionHandle, + segment_file: HlsSegmentFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> Response { + match serve_hls_segment_cache_outcome(segment_cache, session, segment_file, range_header, context).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => hls_resource_failure_default_response(failure), + } +} + +/// Serves a committed Ready segment or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_segment_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + segment_file: HlsSegmentFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + match lookup_segment_cache_object(&session, &segment_file, &context.hls_access_lease_id).await { + CacheObjectLookup::Ready(object) => { + HlsResourceServeOutcome::Ready(serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext::from_response_context(context), + ) + .await) + } + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +/// Serves a committed Ready EXT-X-MAP object from the HLS cache. +pub async fn serve_hls_map_cache_response( + segment_cache: Arc, + session: HlsSessionHandle, + map_file: HlsMapFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> Response { + match serve_hls_map_cache_outcome(segment_cache, session, map_file, range_header, context).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => hls_resource_failure_default_response(failure), + } +} + +/// Serves a committed Ready EXT-X-MAP object or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_map_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + map_file: HlsMapFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + match lookup_map_cache_object(&session, &map_file, &context.hls_access_lease_id).await { + CacheObjectLookup::Ready(object) => { + HlsResourceServeOutcome::Ready(serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext::from_response_context(context), + ) + .await) + } + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +/// Serves a committed Ready transient passthrough full object from the HLS cache. +pub async fn serve_hls_transient_object_cache_response( + segment_cache: Arc, + session: HlsSessionHandle, + resource_file: TransientResourceFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> Response { + match serve_hls_transient_object_cache_outcome(segment_cache, session, resource_file, range_header, context).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => hls_resource_failure_default_response(failure), + } +} + +/// Serves a committed Ready transient passthrough object or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_transient_object_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + resource_file: TransientResourceFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + match lookup_transient_object_cache_object(&session, &resource_file, &context.hls_access_lease_id, context.now_ms) + .await + { + CacheObjectLookup::Ready(object) => { + HlsResourceServeOutcome::Ready(serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext::from_response_context(context), + ) + .await) + } + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +fn hls_resource_failure_default_response(failure: HlsResourceServeFailure) -> Response { + match failure { + HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms } => { + service_unavailable_not_ready_response(retry_after_ms) + } + HlsResourceServeFailure::Missing + | HlsResourceServeFailure::Expired + | HlsResourceServeFailure::PermanentFailed { .. } => StatusCode::NOT_FOUND.into_response(), + } +} + +async fn serve_cache_object( + segment_cache: Arc, + object: CacheObject, + range_header: Option, + context: CacheObjectServeContext, +) -> Response +where + K: super::HlsCacheObjectKey + Send + Sync + 'static, +{ + let guard = CacheReadGuard::new(Arc::clone(&object.access), context.now_ms); + if let Some(repair_context) = object.repair_context.clone() { + if let Err(err) = context + .segment_repair + .repair_ready_cache_hit(&segment_cache, &object.key, repair_context) + .await + { + debug!( + "HLS segment repair skipped for ready cache hit: session={} resource={} error={err}", + object.log_context.session, object.log_context.resource_id + ); + } + } + let metadata_started_at = Instant::now(); + let metadata = match segment_cache.metadata(&object.key).await { + Ok(Some(metadata)) => metadata, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(err) if err.kind() == io::ErrorKind::NotFound => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + log_body_reader_wait_if_slow(&object.log_context, "metadata", metadata_started_at.elapsed().as_millis()); + + let body_id = next_hls_body_log_id(); + let range = resolve_range(range_header.as_ref(), metadata.size); + let (status, start, end, content_length) = match range { + RangeDecision::Full => { + if metadata.size == 0 { + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + } + return empty_ok_response(&object.content_type, context.cache_duration_seconds); + } + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + } + debug!( + "HLS cache response prepared: body_id={} lease={} session={} resource={} source=cache range=full content_length={} content_type={}", + body_id, + object.log_context.lease, + object.log_context.session, + object.log_context.resource_id, + metadata.size, + object.content_type + ); + (StatusCode::OK, 0, metadata.size - 1, metadata.size) + } + RangeDecision::Partial { start, end, length } => { + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + metrics.record_cache_range_hit(); + } + debug!( + "HLS cache response prepared: body_id={} lease={} session={} resource={} source=cache range={start}-{end} content_length={length} content_type={}", + body_id, + object.log_context.lease, + object.log_context.session, + object.log_context.resource_id, + object.content_type + ); + (StatusCode::PARTIAL_CONTENT, start, end, length) + } + RangeDecision::Unsatisfiable => return range_not_satisfiable_response(metadata.size), + }; + + let file_started_at = Instant::now(); + let file = match segment_cache.open_range(&object.key, start).await { + Ok(file) => file, + Err(err) if err.kind() == io::ErrorKind::NotFound => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + log_body_reader_wait_if_slow(&object.log_context, "file", file_started_at.elapsed().as_millis()); + if let Some(marker) = &context.media_activity_marker { + marker.mark_at(context.now_ms).await; + } + + let stream = ReaderStream::new(file.take(content_length)); + let body_context = CacheBodyLogContext { + session: object.log_context.session.clone(), + resource_id: object.log_context.resource_id.clone(), + object_kind: object.log_context.object_kind, + source: object.log_context.body_source, + content_length, + }; + let stream = ActiveReaderStream::new( + Box::pin(stream), + guard, + body_context, + context.qos_meter, + context.media_activity_marker, + ); + let mut response = Response::new(Body::from_stream(stream)); + *response.status_mut() = status; + + let headers = response.headers_mut(); + insert_header_value(headers, header::CONTENT_TYPE, &object.content_type); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + insert_u64_header(headers, header::CONTENT_LENGTH, content_length); + insert_cache_control(headers, context.cache_duration_seconds); + if status == StatusCode::PARTIAL_CONTENT { + insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes {start}-{end}/{}", metadata.size)); + } + mark_response_as_uncompressed(&mut response); + response +} + +async fn lookup_segment_cache_object( + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + hls_access_lease_id: &HlsAccessLeaseId, +) -> CacheObjectLookup { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + }; + if entry.proxy_file_ext != segment_file.extension { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + } + match entry.status { + SegmentCacheStatus::Ready { .. } => {} + SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Discovered => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }); + } + SegmentCacheStatus::FailedRetryable { retry_after_ms, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); + } + SegmentCacheStatus::FailedPermanent { status, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); + } + SegmentCacheStatus::Expired => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + } + CacheObjectLookup::Ready(CacheObject { + key: entry.cache_key.clone(), + access: Arc::clone(&entry.access), + content_type: entry.content_type.clone(), + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + session: safe_proxy_session_id(&session.proxy_session_id), + resource_id: format!("{:06}", segment_file.proxy_seq), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: Some(HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + proxy_session_id: session.proxy_session_id.clone(), + hls_access_lease_id: Some(hls_access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: segment_file.proxy_seq }, + resource_id: format!("{:06}", segment_file.proxy_seq), + file_ext: entry.proxy_file_ext.clone(), + // Cache-hit repair validation may carry the concrete fetch URL as diagnostic metadata only. + origin_fetch_uri_for_diagnostics: entry + .origin_fetch_ref + .as_ref() + .map(|fetch_ref| fetch_ref.resolved_origin_url.clone()) + .unwrap_or_default(), + media_sequence: Some(entry.origin_key.origin_seq), + discontinuity_sequence: Some(session.discontinuity_sequence), + complete_object: entry.origin_byte_range.is_none(), + encrypted: false, + custom_response: false, + }), + }) +} + +async fn lookup_map_cache_object( + session: &HlsSessionHandle, + map_file: &HlsMapFile, + hls_access_lease_id: &HlsAccessLeaseId, +) -> CacheObjectLookup { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let Some(entry) = session.maps.get(&ProxyMapId(map_file.proxy_map_id)) else { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + }; + if entry.proxy_file_ext != map_file.extension { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + } + match entry.status { + MapCacheStatus::Ready { .. } => {} + MapCacheStatus::Fetching { .. } + | MapCacheStatus::Queued { .. } + | MapCacheStatus::Discovered => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }); + } + MapCacheStatus::FailedRetryable { retry_after_ms, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); + } + MapCacheStatus::FailedPermanent { status, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); + } + MapCacheStatus::Expired => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + } + CacheObjectLookup::Ready(CacheObject { + key: entry.cache_key.clone(), + access: Arc::clone(&entry.access), + content_type: entry.content_type.clone(), + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + session: safe_proxy_session_id(&session.proxy_session_id), + resource_id: format!("map:{:06}", map_file.proxy_map_id), + object_kind: "Map", + body_source: "normal", + }, + repair_context: None, + }) +} + +async fn lookup_transient_object_cache_object( + session: &HlsSessionHandle, + resource_file: &TransientResourceFile, + hls_access_lease_id: &HlsAccessLeaseId, + now_ms: u64, +) -> CacheObjectLookup { + let mut session = session.write().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let proxy_session_id = session.proxy_session_id.clone(); + let key = super::TransientPassthroughState::transient_object_key( + &proxy_session_id, + &resource_file.resource_id, + resource_file.extension.clone(), + ); + let resource_kind = session.transient.resources.get(&resource_file.resource_id).map(|resource| resource.kind); + let Some(entry) = session.transient.ready_object(&key, now_ms) else { + return match session.transient.object_cache.get(&key).map(|entry| &entry.status) { + Some(super::TransientObjectCacheStatus::Fetching { .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }) + } + Some(super::TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: *retry_after_ms, + }) + } + Some(super::TransientObjectCacheStatus::FailedPermanent { status, .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status: *status }) + } + Some(super::TransientObjectCacheStatus::Ready { .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::Expired) + } + None => CacheObjectLookup::Failure(HlsResourceServeFailure::Missing), + }; + }; + CacheObjectLookup::Ready(CacheObject { + key: entry.key, + access: Arc::clone(&entry.access), + content_type: entry.content_type, + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + session: safe_proxy_session_id(&proxy_session_id), + resource_id: resource_file.resource_id.0.clone(), + object_kind: transient_body_object_kind(resource_kind, &resource_file.extension), + body_source: "transient", + }, + repair_context: Some(HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Transient, + proxy_session_id, + hls_access_lease_id: Some(hls_access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Transient { + resource_id: resource_file.resource_id.0.clone(), + }, + resource_id: resource_file.resource_id.0.clone(), + file_ext: resource_file.extension.clone(), + origin_fetch_uri_for_diagnostics: resource_file.resource_id.0.clone(), + media_sequence: None, + discontinuity_sequence: None, + complete_object: true, + encrypted: false, + custom_response: false, + }), + }) +} + +fn empty_ok_response(content_type: &str, cache_duration_seconds: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::OK; + let headers = response.headers_mut(); + insert_header_value(headers, header::CONTENT_TYPE, content_type); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); + insert_cache_control(headers, cache_duration_seconds); + mark_response_as_uncompressed(&mut response); + response +} + +fn range_not_satisfiable_response(full_size: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE; + let headers = response.headers_mut(); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); + insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes */{full_size}")); + mark_response_as_uncompressed(&mut response); + response +} + +fn service_unavailable_not_ready_response(retry_after_ms: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + let headers = response.headers_mut(); + insert_header_value( + headers, + header::RETRY_AFTER, + &super::retry_after_secs_from_ms(retry_after_ms).to_string(), + ); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + mark_response_as_uncompressed(&mut response); + response +} + +fn resolve_range(range_header: Option<&HeaderValue>, full_size: u64) -> RangeDecision { + let Some(range_header) = range_header else { + return RangeDecision::Full; + }; + let Ok(range_header) = range_header.to_str() else { + return RangeDecision::Full; + }; + let Some(range_spec) = range_header.strip_prefix("bytes=") else { + return RangeDecision::Full; + }; + if range_spec.contains(',') || full_size == 0 { + return RangeDecision::Unsatisfiable; + } + + let Some((start, end)) = range_spec.split_once('-') else { + return RangeDecision::Unsatisfiable; + }; + if start.is_empty() { + return resolve_suffix_range(end, full_size); + } + resolve_start_range(start, end, full_size) +} + +fn resolve_suffix_range(suffix_length: &str, full_size: u64) -> RangeDecision { + let Ok(suffix_length) = suffix_length.parse::() else { + return RangeDecision::Unsatisfiable; + }; + if suffix_length == 0 { + return RangeDecision::Unsatisfiable; + } + let length = suffix_length.min(full_size); + let start = full_size - length; + let end = full_size - 1; + RangeDecision::Partial { start, end, length } +} + +fn resolve_start_range(start: &str, end: &str, full_size: u64) -> RangeDecision { + let Ok(start) = start.parse::() else { + return RangeDecision::Unsatisfiable; + }; + if start >= full_size { + return RangeDecision::Unsatisfiable; + } + let end = if end.is_empty() { + full_size - 1 + } else { + let Ok(parsed_end) = end.parse::() else { + return RangeDecision::Unsatisfiable; + }; + parsed_end.min(full_size - 1) + }; + if end < start { + return RangeDecision::Unsatisfiable; + } + RangeDecision::Partial { start, end, length: end - start + 1 } +} + +fn insert_cache_control(headers: &mut axum::http::HeaderMap, cache_duration_seconds: u64) { + insert_header_value( + headers, + header::CACHE_CONTROL, + &format!("public, max-age={cache_duration_seconds}, immutable"), + ); +} + +fn insert_u64_header(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: u64) { + insert_header_value(headers, name, &value.to_string()); +} + +fn insert_header_value(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: &str) { + if let Ok(value) = HeaderValue::from_str(value) { + headers.insert(name, value); + } +} + +struct ActiveReaderStream { + inner: Pin> + Send>>, + _guard: CacheReadGuard, + context: CacheBodyLogContext, + started_at: Instant, + last_yield_at: Instant, + send_deadline: Pin>, + max_idle_ms: u128, + completed_logged: bool, + finished: bool, + bytes_yielded: u64, + meter: Arc>, + media_activity_marker: Option, +} + +impl ActiveReaderStream { + fn new( + inner: Pin> + Send>>, + guard: CacheReadGuard, + context: CacheBodyLogContext, + meter: Arc>, + media_activity_marker: Option, + ) -> Self { + Self { + inner, + _guard: guard, + context, + started_at: Instant::now(), + last_yield_at: Instant::now(), + send_deadline: Box::pin(sleep(hls_client_body_send_deadline())), + max_idle_ms: 0, + completed_logged: false, + finished: false, + bytes_yielded: 0, + meter, + media_activity_marker, + } + } + + fn log_completed(&mut self, outcome: &'static str) { + if self.completed_logged { + return; + } + self.completed_logged = true; + debug!( + "{} '{}' body completed: session={} source={} elapsed_s={:.3} idle_max_s={:.3} bytes={}/{} outcome={}", + self.context.object_kind, + self.context.resource_id, + self.context.session, + self.context.source, + duration_secs(self.started_at.elapsed().as_millis()), + duration_secs(self.max_idle_ms), + self.bytes_yielded, + self.context.content_length, + outcome + ); + if let Some(marker) = &self.media_activity_marker { + marker.spawn_mark_now(); + } + } +} + +impl Stream for ActiveReaderStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.finished { + return Poll::Ready(None); + } + if self.send_deadline.as_mut().poll(cx).is_ready() { + self.finished = true; + self.log_completed("timeout"); + return Poll::Ready(Some(Err(io::Error::new( + io::ErrorKind::TimedOut, + "hls client body send timed out", + )))); + } + match self.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + refresh_hls_client_body_send_deadline(self.send_deadline.as_mut()); + let idle_ms = self.last_yield_at.elapsed().as_millis(); + self.max_idle_ms = self.max_idle_ms.max(idle_ms); + self.last_yield_at = Instant::now(); + self.bytes_yielded = self.bytes_yielded.saturating_add(chunk.len() as u64); + if let Some(meter) = self.meter.load_full() { + meter.record_bytes(chunk.len() as u64); + } + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Some(Err(err))) => { + self.finished = true; + self.log_completed("error"); + Poll::Ready(Some(Err(err))) + } + Poll::Ready(None) => { + self.finished = true; + self.log_completed("ok"); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl Drop for ActiveReaderStream { + fn drop(&mut self) { + let outcome = if self.bytes_yielded >= self.context.content_length { "ok" } else { "drop" }; + self.log_completed(outcome); + } +} + +struct CacheReadGuard { + access: Arc, +} + +impl CacheReadGuard { + fn new(access: Arc, now_ms: u64) -> Self { + access.reader_started(now_ms); + Self { access } + } +} + +impl Drop for CacheReadGuard { + fn drop(&mut self) { self.access.reader_finished(); } +} + +fn log_body_reader_wait_if_slow(context: &CacheObjectLogContext, wait_for: &'static str, elapsed_ms: u128) { + if elapsed_ms < BODY_READER_WAIT_LOG_THRESHOLD_MS { + return; + } + debug!( + "HLS cache reader wait: lease={} session={} resource={} wait_for={} elapsed_ms={}", + context.lease, context.session, context.resource_id, wait_for, elapsed_ms + ); +} + +fn duration_secs(elapsed_ms: u128) -> f64 { + Duration::from_millis(u64::try_from(elapsed_ms).unwrap_or(u64::MAX)).as_secs_f64() +} + +fn transient_body_object_kind(resource_kind: Option, extension: &str) -> &'static str { + match resource_kind { + Some(TransientResourceKind::Key) => "Key", + Some(TransientResourceKind::Map) => "Map", + Some(TransientResourceKind::Segment | TransientResourceKind::Other) => "Segment", + None => { + if extension.eq_ignore_ascii_case("key") { "Key" } else { "Segment" } + } + } +} + +fn next_hls_body_log_id() -> String { + let value = NEXT_HLS_BODY_LOG_ID.fetch_add(1, Ordering::Relaxed); + format!("{value:08x}") +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{ + resolve_range, serve_cache_object, transient_body_object_kind, CacheObject, CacheObjectLogContext, + CacheObjectServeContext, HlsMediaActivityMarker, RangeDecision, + }; + use crate::{ + api::model::{ + CacheAccessState, HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSession, HlsSessionKey, + ProxySessionId, SegmentCacheKey, TransientResourceKind, + }, + model::HlsSegmentRepairConfig, + }; + use arc_swap::ArcSwapOption; + use axum::http::{header, HeaderValue, StatusCode}; + use bytes::Bytes; + use http_body_util::BodyExt; + use std::{sync::Arc, time::Duration}; + use tokio::sync::RwLock; + use shared::model::HlsSegmentRepairMode; + + fn header(value: &str) -> HeaderValue { HeaderValue::from_str(value).expect("valid header") } + + fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) + } + + #[test] + fn range_parser_resolves_open_ended_range() { + assert_eq!( + resolve_range(Some(&header("bytes=4-")), 10), + RangeDecision::Partial { start: 4, end: 9, length: 6 } + ); + } + + #[test] + fn range_parser_resolves_closed_range() { + assert_eq!( + resolve_range(Some(&header("bytes=2-5")), 10), + RangeDecision::Partial { start: 2, end: 5, length: 4 } + ); + } + + #[test] + fn range_parser_resolves_suffix_range() { + assert_eq!( + resolve_range(Some(&header("bytes=-3")), 10), + RangeDecision::Partial { start: 7, end: 9, length: 3 } + ); + } + + #[test] + fn range_parser_rejects_unsatisfiable_and_multi_ranges() { + assert_eq!(resolve_range(Some(&header("bytes=20-")), 10), RangeDecision::Unsatisfiable); + assert_eq!(resolve_range(Some(&header("bytes=0-1,4-5")), 10), RangeDecision::Unsatisfiable); + } + + #[test] + fn range_parser_rejects_malformed_bytes_ranges() { + assert_eq!(resolve_range(Some(&header("bytes=abc")), 10), RangeDecision::Unsatisfiable); + assert_eq!(resolve_range(Some(&header("bytes=a-b")), 10), RangeDecision::Unsatisfiable); + } + + #[test] + fn range_parser_ignores_unknown_units() { + assert_eq!(resolve_range(Some(&header("items=0-1")), 10), RangeDecision::Full); + } + + #[test] + fn transient_body_log_kind_uses_resource_kind() { + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Key), "bin"), "Key"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Map), "bin"), "Map"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Segment), "key"), "Segment"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Other), "bin"), "Segment"); + } + + #[test] + fn transient_body_log_kind_falls_back_to_key_extension() { + assert_eq!(transient_body_object_kind(None, "key"), "Key"); + assert_eq!(transient_body_object_kind(None, "KEY"), "Key"); + assert_eq!(transient_body_object_kind(None, "ts"), "Segment"); + } + + #[test] + fn temporary_unavailable_response_uses_concrete_retry_after() { + let response = + super::hls_resource_failure_default_response(super::HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: 2_500, + }); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry-after"), "3"); + } + + #[tokio::test] + async fn cache_hit_bodies_use_independent_readers_for_same_object() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); + segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + let access = Arc::new(CacheAccessState::new()); + let object = CacheObject { + key, + access: Arc::clone(&access), + content_type: "video/mp2t".to_string(), + log_context: CacheObjectLogContext { + lease: "lease-a".to_string(), + session: "proxy-se...".to_string(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: None, + }; + + let first = serve_cache_object( + Arc::clone(&segment_cache), + object.clone(), + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: None, + now_ms: 1, + }, + ) + .await; + let second = serve_cache_object( + segment_cache, + CacheObject { + log_context: CacheObjectLogContext { + lease: "lease-b".to_string(), + session: "proxy-se...".to_string(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + ..object + }, + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: None, + now_ms: 2, + }, + ) + .await; + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(access.active_readers(), 2); + + let (first_body, second_body) = tokio::join!(first.into_body().collect(), second.into_body().collect(),); + + assert_eq!(first_body.expect("first body").to_bytes(), Bytes::from_static(b"0123456789")); + assert_eq!(second_body.expect("second body").to_bytes(), Bytes::from_static(b"0123456789")); + assert_eq!(access.active_readers(), 0); + assert_eq!(access.last_accessed_at_ms(), 2); + } + + #[tokio::test] + async fn cache_body_marks_media_activity_at_start_and_body_end() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); + segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + let access = Arc::new(CacheAccessState::new()); + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + let marker = HlsMediaActivityMarker::new(Arc::clone(&hls_proxy), Arc::clone(&session)); + + let response = serve_cache_object( + segment_cache, + CacheObject { + key, + access, + content_type: "video/mp2t".to_string(), + log_context: CacheObjectLogContext { + lease: "lease-a".to_string(), + session: "proxy-se...".to_string(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: None, + }, + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: Some(marker), + now_ms: 1_000, + }, + ) + .await; + + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(1_000)); + + assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"0123456789")); + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!( + session.read().await.activity.last_authorized_media_at_ms.expect("media activity") >= 1_000, + "body completion should not move media activity backwards" + ); + } +} diff --git a/backend/src/api/model/hls_cache/segment_fetcher.rs b/backend/src/api/model/hls_cache/segment_fetcher.rs new file mode 100644 index 000000000..85570c547 --- /dev/null +++ b/backend/src/api/model/hls_cache/segment_fetcher.rs @@ -0,0 +1,1515 @@ +#![allow(clippy::large_futures)] + +use super::{ + begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, classify_hls_backpressure, + finish_hls_origin_account_io, hls_object_body_deadline, + run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, + HlsAccessLeaseChannelUnavailableReason, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, + HlsBoundAccountAcquireErrorKind, HlsCacheMetrics, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, + HlsOriginIoContext, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentFailureObject, + HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, HlsSessionHandle, OriginSegmentFetchRef, SegmentCacheKey, SegmentCacheStatus, + SegmentFetchPriority, +}; +use crate::model::HlsCacheConfig; +use crate::processing::parser::hls::origin_manifest::ParsedByteRange; +use shared::model::{HlsSegmentRepairMode, HlsStripMode}; +use arc_swap::ArcSwap; +use axum::http::HeaderMap; +use futures::{FutureExt, TryStreamExt}; +use log::{debug, warn}; +use reqwest::Client; +use std::{fmt, io, sync::Arc, time::Duration}; +use tokio::{ + sync::{OwnedSemaphorePermit, RwLock, Semaphore}, + time::timeout, +}; +use tokio_util::io::StreamReader; + +const DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES: usize = 64; +const DEFAULT_MAX_SESSION_SEGMENT_FETCHES: usize = 2; +const DEFAULT_MAX_PREFETCH_QUEUE_DEPTH: usize = 6; +const DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS: u64 = 10_000; + +/// Runtime policy for bounded live HLS segment origin fetches. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct SegmentFetchPolicy { + pub max_global_segment_fetches: usize, + pub max_session_segment_fetches: usize, + pub max_prefetch_queue_depth: usize, + pub origin_segment_timeout_ms: u64, + pub effective_repair_postprocess_timeout_ms: u64, + pub retry_delays_ms: [u64; 5], + pub retry_jitter_max_ms: u64, + pub permanent_failure_segment_threshold: u32, +} + +impl SegmentFetchPolicy { + pub fn from_config(config: &HlsCacheConfig) -> Self { + let postprocess_enabled = (config.segment_repair.max_level != HlsSegmentRepairMode::Off + && config.segment_repair.apply_to_first_segments > 0) + || config.segment_repair.corrupt_segment_watchdog.mode.is_enabled(); + Self { + max_global_segment_fetches: config.max_concurrent_segment_fetches_global.max(1), + max_session_segment_fetches: config.max_concurrent_segment_fetches_per_session.max(1), + max_prefetch_queue_depth: config.max_segments_prefetch, + origin_segment_timeout_ms: config.origin_segment_timeout_ms.max(1), + effective_repair_postprocess_timeout_ms: if postprocess_enabled { + config.segment_repair.postprocess_timeout_ms.max(100) + } else { + 0 + }, + permanent_failure_segment_threshold: permanent_failure_segment_threshold_from_config(config), + ..Self::default() + } + } + + pub fn demand_wait_timeout(&self) -> Duration { + let attempts = u64::try_from(self.retry_delays_ms.len()).unwrap_or(u64::MAX); + let per_attempt_budget = + self.origin_segment_timeout_ms.saturating_add(self.effective_repair_postprocess_timeout_ms); + let retry_delay_budget = self.retry_delays_ms.iter().copied().fold(0_u64, u64::saturating_add); + let jitter_budget = attempts.saturating_mul(self.retry_jitter_max_ms); + Duration::from_millis( + attempts + .saturating_mul(per_attempt_budget) + .saturating_add(retry_delay_budget) + .saturating_add(jitter_budget) + .saturating_add(1_000), + ) + } + + pub fn retry_delay_ms(&self, attempt_index: usize) -> u64 { + let base_delay_ms = self.retry_delays_ms[attempt_index]; + if self.retry_jitter_max_ms == 0 { + return base_delay_ms; + } + let jitter_ms = fastrand::u64(0..=self.retry_jitter_max_ms); + if fastrand::bool() { + base_delay_ms.saturating_sub(jitter_ms) + } else { + base_delay_ms.saturating_add(jitter_ms) + } + } +} + +impl Default for SegmentFetchPolicy { + fn default() -> Self { + Self { + max_global_segment_fetches: DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES, + max_session_segment_fetches: DEFAULT_MAX_SESSION_SEGMENT_FETCHES, + max_prefetch_queue_depth: DEFAULT_MAX_PREFETCH_QUEUE_DEPTH, + origin_segment_timeout_ms: DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS, + effective_repair_postprocess_timeout_ms: 0, + retry_delays_ms: [0, 100, 250, 500, 750], + retry_jitter_max_ms: 100, + permanent_failure_segment_threshold: 3, + } + } +} + +fn permanent_failure_segment_threshold_from_config(config: &HlsCacheConfig) -> u32 { + let configured_strip_segments = match config.strip.mode { + HlsStripMode::Segments => u32::try_from(config.strip.value).unwrap_or(u32::MAX.saturating_sub(3)), + HlsStripMode::Seconds => 0, + }; + 3_u32.saturating_add(configured_strip_segments) +} + +/// Shared context required to schedule a segment fetch without holding session locks. +#[derive(Clone)] +pub struct SegmentFetchContext { + pub session: HlsSessionHandle, + pub segment_cache: Arc, + pub segment_repair: Arc, + pub repair_access_lease_id: Option, + pub headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub client: Client, + pub no_redirect_client: Client, + pub use_manual_redirects: bool, + pub origin_io: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SegmentDemandFetchOutcome { + Ready, + QueuedOrFetching, + NotFound, + Unavailable, + TimedOut, +} + +#[derive(Clone)] +struct SegmentFetchSnapshot { + proxy_seq: u64, + proxy_seq_log: String, + cache_key: SegmentCacheKey, + fetch_ref: OriginSegmentFetchRef, + proxy_file_ext: String, + origin_seq: u64, + complete_object: bool, +} + +struct SegmentFetchCommit { + content_length: u64, + generation_valid: bool, +} + +#[derive(Clone, Copy)] +struct SegmentOriginWorkFinish { + generation_valid: bool, + refresh_reservation: bool, +} + +impl fmt::Debug for SegmentFetchSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentFetchSnapshot") + .field("proxy_seq", &self.proxy_seq) + .field("proxy_seq_log", &self.proxy_seq_log) + .field("cache_key", &self.cache_key) + .field("fetch_ref", &self.fetch_ref) + .field("proxy_file_ext", &self.proxy_file_ext) + .field("origin_seq", &self.origin_seq) + .field("complete_object", &self.complete_object) + .finish() + } +} + +type SegmentFetchError = HlsOriginResourceFetchError; + +#[derive(Clone)] +struct SegmentWorkerRuntime { + global_semaphore: Arc, + policy: SegmentFetchPolicy, +} + +impl SegmentWorkerRuntime { + fn new(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { Self { global_semaphore, policy } } +} + +/// Bounded scheduler for live HLS segment demand fetches and prefetches. +pub struct HlsSegmentWorkerPool { + runtime: ArcSwap, + access_leases: Arc>, + metrics: Arc, +} + +impl HlsSegmentWorkerPool { + pub fn new(policy: SegmentFetchPolicy) -> Self { + let global_semaphore = Arc::new(Semaphore::new(policy.max_global_segment_fetches)); + Self::with_global_semaphore(policy, global_semaphore) + } + + pub fn with_global_semaphore(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { + Self::with_global_semaphore_and_metrics( + policy, + global_semaphore, + Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + Arc::new(HlsCacheMetrics::default()), + ) + } + + pub fn with_global_semaphore_and_metrics( + policy: SegmentFetchPolicy, + global_semaphore: Arc, + access_leases: Arc>, + metrics: Arc, + ) -> Self { + Self { + runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)), + access_leases, + metrics, + } + } + + pub fn update_config(&self, policy: SegmentFetchPolicy, global_semaphore: Arc) { + self.runtime.store(Arc::new(SegmentWorkerRuntime::new(policy, global_semaphore))); + } + + pub fn policy(&self) -> SegmentFetchPolicy { self.runtime.load().policy.clone() } + + pub fn access_leases(&self) -> &Arc> { &self.access_leases } + + pub fn metrics(&self) -> &Arc { &self.metrics } + + pub async fn classify_backpressure(&self, session: &HlsSessionHandle) -> HlsBackpressureState { + let session = session.read().await; + self.classify_backpressure_for_session(&session) + } + + pub fn classify_backpressure_for_session(&self, session: &super::HlsSession) -> HlsBackpressureState { + let runtime = self.runtime.load(); + classify_hls_backpressure( + session, + runtime.global_semaphore.available_permits(), + runtime.policy.max_session_segment_fetches, + ) + } + + pub async fn demand_fetch_and_wait( + self: &Arc, + context: SegmentFetchContext, + segment_file: &HlsSegmentFile, + now_ms: u64, + ) -> SegmentDemandFetchOutcome { + let wait_timeout = self.runtime.load().policy.demand_wait_timeout(); + let notifier = { + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() { + return SegmentDemandFetchOutcome::NotFound; + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return SegmentDemandFetchOutcome::NotFound; + }; + if entry.proxy_file_ext != segment_file.extension { + return SegmentDemandFetchOutcome::NotFound; + } + match entry.status { + SegmentCacheStatus::Ready { .. } => return SegmentDemandFetchOutcome::Ready, + SegmentCacheStatus::Fetching { .. } => { + session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() + } + SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. } => { + if entry.origin_fetch_ref.is_none() { + return SegmentDemandFetchOutcome::Unavailable; + } + let backpressure = self.classify_backpressure_for_session(&session); + if !backpressure.allows_new_demand_fetch() { + warn!( + "HLS segment demand fetch skipped by backpressure: session={} source=normal resource={:06} state={backpressure:?}", + super::safe_proxy_session_id(&session.proxy_session_id), + segment_file.proxy_seq + ); + return SegmentDemandFetchOutcome::Unavailable; + } + session.queue_segment_fetch_candidate(segment_file.proxy_seq, SegmentFetchPriority::Demand, now_ms); + self.metrics.record_demand_fetch_started(); + debug!( + "HLS segment demand fetch started: session={} source=normal resource={:06}", + super::safe_proxy_session_id(&session.proxy_session_id), + segment_file.proxy_seq + ); + session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() + } + SegmentCacheStatus::FailedRetryable { .. } => { + return SegmentDemandFetchOutcome::TimedOut; + } + SegmentCacheStatus::FailedPermanent { .. } | SegmentCacheStatus::Expired => { + return SegmentDemandFetchOutcome::Unavailable; + } + } + }; + + self.wake_scheduler(context.clone(), now_ms).await; + + if timeout(wait_timeout, notifier.notified()).await.is_err() { + return SegmentDemandFetchOutcome::TimedOut; + } + + let session = context.session.read().await; + if session.is_gc_marked_for_removal() { + return SegmentDemandFetchOutcome::NotFound; + } + match session.segments.get(&segment_file.proxy_seq).map(|entry| &entry.status) { + Some(SegmentCacheStatus::Ready { .. }) => SegmentDemandFetchOutcome::Ready, + Some(SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. }) => { + SegmentDemandFetchOutcome::QueuedOrFetching + } + Some( + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Expired + | SegmentCacheStatus::FailedPermanent { .. }, + ) => SegmentDemandFetchOutcome::Unavailable, + Some(SegmentCacheStatus::FailedRetryable { .. }) => SegmentDemandFetchOutcome::TimedOut, + None => SegmentDemandFetchOutcome::NotFound, + } + } + + pub async fn wake_scheduler(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + loop { + let runtime = self.runtime.load_full(); + let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { + return; + }; + let Some(snapshot) = self.next_fetch_snapshot(&context, now_ms, &runtime.policy).await else { + drop(permit); + return; + }; + + let worker = Arc::clone(self); + let task_context = context.clone(); + tokio::spawn(async move { + worker.fetch_one_segment(task_context, snapshot, runtime.policy.clone(), permit).await; + }); + } + } + + async fn next_fetch_snapshot( + &self, + context: &SegmentFetchContext, + now_ms: u64, + policy: &SegmentFetchPolicy, + ) -> Option { + let (proxy_session_id, gc_marked_for_removal) = { + let session = context.session.read().await; + (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + }; + if gc_marked_for_removal { + return None; + } + let has_usable_access_lease = + self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms); + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() { + return None; + } + if session.active_segment_fetches >= policy.max_session_segment_fetches { + return None; + } + + while let Some((proxy_seq, priority)) = session.segment_prefetch_queue.pop_next() { + let Some(entry) = session.segments.get_mut(&proxy_seq) else { + continue; + }; + if !matches!(entry.status, SegmentCacheStatus::Queued { .. }) { + continue; + } + if priority != SegmentFetchPriority::Demand && !has_usable_access_lease { + entry.status = SegmentCacheStatus::Discovered; + continue; + } + let Some(fetch_ref) = entry.origin_fetch_ref.clone() else { + continue; + }; + if !fetch_ref.is_valid_at(now_ms) { + continue; + } + + let cache_key = entry.cache_key.clone(); + let proxy_file_ext = entry.proxy_file_ext.clone(); + let origin_seq = entry.origin_key.origin_seq; + let complete_object = entry.origin_byte_range.is_none(); + entry.status = SegmentCacheStatus::Fetching { priority, started_at_ms: now_ms }; + session.active_segment_fetches = session.active_segment_fetches.saturating_add(1); + let proxy_seq_log = format!("{proxy_seq:06}"); + debug!( + "HLS segment fetch started: session={} source=normal resource={} priority={priority:?}", + super::safe_proxy_session_id(&session.proxy_session_id), + proxy_seq_log + ); + return Some(SegmentFetchSnapshot { + proxy_seq, + proxy_seq_log, + cache_key, + fetch_ref, + proxy_file_ext, + origin_seq, + complete_object, + }); + } + + None + } + + #[allow(clippy::too_many_lines)] + async fn fetch_one_segment( + self: Arc, + context: SegmentFetchContext, + snapshot: SegmentFetchSnapshot, + policy: SegmentFetchPolicy, + permit: OwnedSemaphorePermit, + ) { + let result = fetch_segment_into_cache(&context, &snapshot, &policy).await; + let finished_at_ms = current_time_millis(); + let generation_valid = result.as_ref().map_or(true, |commit| commit.generation_valid); + let fetch_succeeded = result.is_ok(); + let (notifier, response_flag_reason) = { + let mut session = context.session.write().await; + session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); + let mut response_flag_reason = None; + match result { + Ok(commit) => { + let content_length = commit.content_length; + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::Ready { content_length, ready_at_ms: finished_at_ms }; + } + let reset_failures = session.record_successful_segment_fetch(); + self.metrics.record_segment_cached(); + debug!( + "HLS segment cached: session={} source=normal resource={} content_length={content_length}", + super::safe_proxy_session_id(&session.proxy_session_id), + snapshot.proxy_seq_log + ); + if let Some(reset_failures) = reset_failures { + debug!( + "HLS segment temporary failure counter reset: session={} previous_failures={reset_failures}", + super::safe_proxy_session_id(&session.proxy_session_id) + ); + } + } + Err(err) => { + let mut invalidate_queued_origin_work = false; + let status = if err.retryable_failure() { + let threshold = + session.segment_temporary_failure_threshold(policy.permanent_failure_segment_threshold); + match session.record_temporary_segment_fetch_failure( + finished_at_ms, + HlsSegmentFailureObject::Normal { + proxy_seq: snapshot.proxy_seq, + origin_seq: snapshot.origin_seq, + }, + threshold, + ) { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { + debug!( + "HLS segment temporary failure counted: session={} object={} failures={} threshold={}", + super::safe_proxy_session_id(&session.proxy_session_id), + snapshot.proxy_seq_log, + failures, + threshold + ); + SegmentCacheStatus::FailedRetryable { + failed_at_ms: finished_at_ms, + retry_after_ms: 1_000, + } + } + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { + warn!( + "HLS segment temporary failure threshold reached: session={} failures={} threshold={}", + super::safe_proxy_session_id(&session.proxy_session_id), + failures, + threshold + ); + invalidate_queued_origin_work = true; + response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::SegmentTemporaryFailureThreshold { + failures, + threshold, + }); + SegmentCacheStatus::FailedPermanent { + failed_at_ms: finished_at_ms, + status: None, + } + } + } + } else { + response_flag_reason = Some(HlsAccessLeaseChannelUnavailableReason::SegmentPermanentFailure { + status: err.permanent_status(), + }); + SegmentCacheStatus::FailedPermanent { + failed_at_ms: finished_at_ms, + status: err.permanent_status(), + } + }; + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = status; + } + if invalidate_queued_origin_work { + session.invalidate_queued_origin_work(); + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::FailedPermanent { + failed_at_ms: finished_at_ms, + status: None, + }; + } + } + } + } + if fetch_succeeded && generation_valid { + let _ = session.render_and_store_manifest(finished_at_ms); + } + (session.segment_fetch_notifiers.remove(&snapshot.proxy_seq), response_flag_reason) + }; + if let Some(reason) = response_flag_reason { + let marked = self.mark_channel_unavailable_for_session(&context.session, finished_at_ms, reason).await; + if marked > 0 { + debug!( + "HLS access leases marked channel unavailable: session={} marked={marked}", + super::safe_proxy_session_id(&context.session.read().await.proxy_session_id) + ); + } + } + if let Some(notifier) = notifier { + notifier.notify_waiters(); + } + drop(permit); + if generation_valid { + self.schedule_wake(context, finished_at_ms); + } + } + + fn schedule_wake(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + let worker = Arc::clone(self); + tokio::spawn(async move { + worker.wake_scheduler(context, now_ms).await; + }); + } + + async fn mark_channel_unavailable_for_session( + &self, + session: &HlsSessionHandle, + now_ms: u64, + reason: HlsAccessLeaseChannelUnavailableReason, + ) -> usize { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + self.access_leases + .write() + .await + .mark_channel_unavailable_for_session(&proxy_session_id, now_ms, reason) + } +} + +impl Default for HlsSegmentWorkerPool { + fn default() -> Self { Self::new(SegmentFetchPolicy::default()) } +} + +async fn fetch_segment_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + fetch_segment_with_retries_into_cache(context, snapshot, policy).await +} + +struct SegmentOriginAttemptGuard { + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +} + +async fn prepare_segment_origin_attempt( + context: SegmentFetchContext, + policy: SegmentFetchPolicy, +) -> Result { + let started_generation = start_segment_origin_work(&context).await; + let binding = + if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; + let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { + if binding.is_detached() { + let _ = finish_segment_origin_work(&context, started_generation).await; + touch_segment_origin_account_binding(&context, false).await; + return Err(SegmentFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); + } + let guard = match begin_hls_origin_account_io_bounded( + origin_io, + &context.session, + binding, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + { + Ok(guard) => guard, + Err(err) => { + let _ = finish_segment_origin_work(&context, started_generation).await; + touch_segment_origin_account_binding(&context, false).await; + return Err(SegmentFetchError::ProviderUnavailable(err)); + } + }; + Some((origin_io.clone(), guard)) + } else { + None + }; + Ok(SegmentOriginAttemptGuard { started_generation, provider_lease }) +} + +async fn finish_segment_origin_attempt( + context: SegmentFetchContext, + guard: SegmentOriginAttemptGuard, +) -> SegmentOriginWorkFinish { + finish_segment_origin_io(&context, guard.started_generation, guard.provider_lease).await +} + +async fn finish_segment_origin_io( + context: &SegmentFetchContext, + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +) -> SegmentOriginWorkFinish { + let origin_work = finish_segment_origin_work(context, started_generation).await; + if let Some((origin_io, guard)) = provider_lease { + finish_hls_origin_account_io( + &origin_io, + &context.session, + guard, + origin_work.generation_valid && origin_work.refresh_reservation, + ) + .await; + touch_segment_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) + .await; + } + origin_work +} + +async fn start_segment_origin_work(context: &SegmentFetchContext) -> Option { + context.origin_io.as_ref()?; + let mut session = context.session.write().await; + Some(session.start_origin_work()) +} + +async fn finish_segment_origin_work( + context: &SegmentFetchContext, + started_generation: Option, +) -> SegmentOriginWorkFinish { + let Some(started_generation) = started_generation else { + return SegmentOriginWorkFinish { generation_valid: true, refresh_reservation: false }; + }; + let mut session = context.session.write().await; + let generation_valid = session.finish_origin_work(started_generation); + let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); + SegmentOriginWorkFinish { generation_valid, refresh_reservation } +} + +async fn touch_segment_origin_account_binding(context: &SegmentFetchContext, reservation_refreshed: bool) { + let mut session = context.session.write().await; + if let Some(binding) = session.origin_account_binding.as_mut() { + let now_ms = current_time_millis(); + binding.last_origin_io_at_ms = Some(now_ms); + if reservation_refreshed { + binding.last_reservation_refresh_at_ms = Some(now_ms); + } + } +} + +#[allow(clippy::too_many_lines)] +async fn fetch_segment_with_retries_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + let headers = build_segment_origin_headers( + &context.headers, + &context.origin_provider_session_headers, + snapshot.fetch_ref.byte_range, + )?; + let target = HlsOriginResourceFetchTarget { + kind: HlsResourceFetchKind::Segment, + source: HlsResourceFetchSource::Normal, + object_id: snapshot.proxy_seq_log.clone(), + origin_url: snapshot.fetch_ref.resolved_origin_url.clone(), + headers, + byte_range_expectation: if snapshot.fetch_ref.byte_range.is_some() { + HlsOriginByteRangeExpectation::PartialContent + } else { + HlsOriginByteRangeExpectation::FullObject + }, + }; + let clients = HlsOriginResourceClients { + client: context.client.clone(), + no_redirect_client: context.no_redirect_client.clone(), + use_manual_redirects: context.use_manual_redirects, + }; + let session_log_id = context.session.read().await.proxy_session_id.0.clone(); + let context = context.clone(); + let snapshot = snapshot.clone(); + let commit_policy = policy.clone(); + let policy_for_prepare = policy.clone(); + let prepare_context = context.clone(); + let cleanup_context = context.clone(); + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + clients, + policy, + &session_log_id, + move |_attempt| { + let context = prepare_context.clone(); + let policy = policy_for_prepare.clone(); + async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + }, + move |guard| { + let context = cleanup_context.clone(); + async move { + finish_segment_origin_attempt(context, guard).await; + } + .boxed() + }, + move |response, _attempt, guard| { + let context = context.clone(); + let snapshot = snapshot.clone(); + let policy = commit_policy.clone(); + async move { + let commit_result = commit_segment_response_into_cache(&context, &snapshot, &policy, response).await; + let origin_work = finish_segment_origin_attempt(context, guard).await; + commit_result.map(|metadata| SegmentFetchCommit { + content_length: metadata.size, + generation_valid: origin_work.generation_valid, + }) + } + .boxed() + }, + ) + .await +} + +async fn commit_segment_response_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, + response: reqwest::Response, +) -> Result { + let deadline = hls_object_body_deadline(policy.origin_segment_timeout_ms); + let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); + let proxy_session_id = context.session.read().await.proxy_session_id.clone(); + let repair_context = HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + proxy_session_id, + hls_access_lease_id: context.repair_access_lease_id.clone(), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: snapshot.proxy_seq }, + resource_id: format!("{:06}", snapshot.proxy_seq), + file_ext: snapshot.proxy_file_ext.clone(), + // Segment repair uses the concrete fetch URL for diagnostics/postprocess metadata only. + origin_fetch_uri_for_diagnostics: snapshot.fetch_ref.resolved_origin_url.clone(), + media_sequence: Some(snapshot.origin_seq), + discontinuity_sequence: None, + complete_object: snapshot.complete_object, + encrypted: false, + custom_response: false, + }; + context + .segment_repair + .commit_origin_response(&context.segment_cache, &snapshot.cache_key, stream_reader, deadline, repair_context) + .await + .map_err(|err| { + if err.kind() == io::ErrorKind::TimedOut { + SegmentFetchError::Timeout + } else { + SegmentFetchError::cache_commit(&err) + } + }) +} + +fn build_segment_origin_headers( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + byte_range: Option, +) -> Result { + build_hls_origin_resource_headers(source_headers, provider_session_headers, byte_range) +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +#[cfg(test)] +mod tests { + use super::{build_segment_origin_headers, SegmentFetchContext, SegmentFetchPolicy, SegmentFetchPriority}; + use crate::{ + api::model::{ + HlsAccessLease, HlsAccessLeaseId, HlsPlaybackFamilyKey, HlsSegmentCache, HlsSegmentFile, + HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, ProxySessionId, + SegmentCacheStatus, + }, + model::HlsSegmentRepairConfig, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + }; + use axum::http::{header, HeaderMap, HeaderValue}; + use std::{ + collections::VecDeque, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Mutex, + }; + use shared::model::HlsSegmentRepairMode; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + #[test] + fn demand_wait_timeout_uses_fetch_attempt_and_postprocess_budget() { + let policy = SegmentFetchPolicy { + origin_segment_timeout_ms: 10_000, + effective_repair_postprocess_timeout_ms: 2_000, + retry_delays_ms: [0, 100, 250, 500, 750], + retry_jitter_max_ms: 100, + ..SegmentFetchPolicy::default() + }; + + assert_eq!(policy.demand_wait_timeout(), Duration::from_millis(63_100)); + } + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) + } + + #[test] + fn segment_origin_headers_remove_client_range_and_force_identity() { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + let headers = build_segment_origin_headers(&headers, &HeaderMap::new(), None).expect("headers should build"); + + assert!(!headers.contains_key(header::RANGE)); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key(header::HOST)); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); + } + + #[test] + fn segment_origin_headers_apply_byterange() { + let headers = build_segment_origin_headers( + &HeaderMap::new(), + &HeaderMap::new(), + Some(crate::processing::parser::hls::origin_manifest::ParsedByteRange { offset: 10, length: 5 }), + ) + .expect("headers should build"); + + assert_eq!(headers.get(header::RANGE).expect("range"), "bytes=10-14"); + } + + #[tokio::test] + async fn segment_fetch_snapshot_uses_concrete_final_segment_fetch_url() { + let manifest = match parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nmedia/seg001.ts\n", + "https://cdn.example.net/live/redirected/playlist.m3u8", + ) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(SegmentFetchPolicy::default().max_prefetch_queue_depth); + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_manifest_prefetch_candidates(10); + } + + let worker = Arc::new(HlsSegmentWorkerPool::new(SegmentFetchPolicy::default())); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease(&worker, &proxy_session_id).await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }; + + let snapshot = worker + .next_fetch_snapshot(&context, 11, &SegmentFetchPolicy::default()) + .await + .expect("segment snapshot"); + + assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/media/seg001.ts"); + assert_eq!(snapshot.fetch_ref.byte_range, None); + assert_eq!(snapshot.origin_seq, 10); + } + + struct TestSegmentServer { + base_url: String, + max_active: Arc, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + } + + impl Drop for TestSegmentServer { + fn drop(&mut self) { self.task.abort(); } + } + + async fn spawn_segment_server(delay_ms: u64) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_active = Arc::clone(&active); + let task_max = Arc::clone(&max_active); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let active = Arc::clone(&task_active); + let max_active = Arc::clone(&task_max); + let requests = Arc::clone(&task_requests); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let mut used = 0_usize; + loop { + let Ok(read) = socket.read(&mut buf[used..]).await else { + return; + }; + if read == 0 { + return; + } + used += read; + if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if used == buf.len() { + return; + } + } + let request = String::from_utf8_lossy(&buf[..used]).to_string(); + requests.lock().await.push(request.clone()); + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(current, Ordering::SeqCst); + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + active.fetch_sub(1, Ordering::SeqCst); + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); + let body = format!("body:{path}"); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), max_active, requests, task } + } + + async fn spawn_sequence_status_server(statuses: Vec) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let max_active = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let statuses = Arc::new(Mutex::new(VecDeque::from(statuses))); + let task_active = Arc::clone(&active); + let task_max = Arc::clone(&max_active); + let task_requests = Arc::clone(&requests); + let task_statuses = Arc::clone(&statuses); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let active = Arc::clone(&task_active); + let max_active = Arc::clone(&task_max); + let requests = Arc::clone(&task_requests); + let statuses = Arc::clone(&task_statuses); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + requests.lock().await.push(request.clone()); + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(current, Ordering::SeqCst); + active.fetch_sub(1, Ordering::SeqCst); + let status = statuses.lock().await.pop_front().unwrap_or(200); + let reason = if status == 200 { "OK" } else { "Error" }; + let body = if status == 200 { "segment-body" } else { "" }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), max_active, requests, task } + } + + async fn spawn_redirect_retry_server() -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let max_active = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + let request_count = Arc::new(AtomicUsize::new(0)); + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let request_count = Arc::clone(&request_count); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); + requests.lock().await.push(path.to_string()); + let count = request_count.fetch_add(1, Ordering::SeqCst); + let response = if path == "/1.ts" && count == 0 { + "HTTP/1.1 302 Found\r\nLocation: /redirected.ts\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string() + } else if path == "/redirected.ts" { + "HTTP/1.1 500 Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string() + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\nsegment-body".to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), max_active, requests, task } + } + + async fn spawn_range_segment_server() -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let max_active = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + task_requests.lock().await.push(request.clone()); + let body = if request.to_ascii_lowercase().contains("range: bytes=10-14") { "seg!!" } else { "bad" }; + let status = if body == "seg!!" { "206 Partial Content" } else { "200 OK" }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Range: bytes 10-14/20\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + + TestSegmentServer { base_url: format!("http://{addr}"), max_active, requests, task } + } + + async fn grant_usable_worker_access_lease(worker: &HlsSegmentWorkerPool, proxy_session_id: &ProxySessionId) { + worker.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + HlsAccessLeaseId("worker-lease".to_string()), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + 10, + 15_000, + )); + } + + async fn fetch_context_with_access_lease( + server: &TestSegmentServer, + temp_dir: &tempfile::TempDir, + policy: &SegmentFetchPolicy, + grant_access_lease: bool, + ) -> (Arc, SegmentFetchContext, HlsSegmentFile) { + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); + session.proxy_next_seq = Some(1); + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{}/1.ts\n#EXTINF:4.0,\n{}/2.ts\n#EXTINF:4.0,\n{}/3.ts\n", + server.base_url, server.base_url, server.base_url + ))) + .expect("manifest maps"); + session.queue_manifest_prefetch_candidates(10); + } + let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); + if grant_access_lease { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease(&worker, &proxy_session_id).await; + } + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: true, + origin_io: None, + }; + (worker, context, HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }) + } + + async fn fetch_context( + server: &TestSegmentServer, + temp_dir: &tempfile::TempDir, + policy: &SegmentFetchPolicy, + ) -> (Arc, SegmentFetchContext, HlsSegmentFile) { + fetch_context_with_access_lease(server, temp_dir, policy, true).await + } + + async fn clear_scheduled_prefetch(context: &SegmentFetchContext, policy: &SegmentFetchPolicy) { + let mut session = context.session.write().await; + session.segment_prefetch_queue = crate::api::model::SegmentPrefetchQueue::new(policy.max_prefetch_queue_depth); + for segment in session.segments.values_mut() { + if !matches!(segment.status, SegmentCacheStatus::Ready { .. }) { + segment.status = SegmentCacheStatus::Discovered; + } + } + } + + #[tokio::test] + async fn demand_fetch_writes_cache_and_sets_ready_after_commit() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + let session = context.session.read().await; + assert!(matches!(session.segments.get(&1).expect("segment").status, SegmentCacheStatus::Ready { .. })); + } + + #[tokio::test] + async fn demand_fetch_is_blocked_for_gc_marked_session() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + context.session.write().await.mark_for_gc_removal(); + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::NotFound); + assert!(server.requests.lock().await.is_empty()); + } + + #[tokio::test] + async fn background_segment_fetch_without_usable_access_lease_resets_queue_without_origin_request() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, _) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; + + worker.wake_scheduler(context.clone(), 20).await; + tokio::time::sleep(Duration::from_millis(20)).await; + + assert!(server.requests.lock().await.is_empty()); + let session = context.session.read().await; + assert_eq!(session.active_segment_fetches, 0); + assert!(session.segment_prefetch_queue.is_empty()); + assert!(session.segments.values().all(|segment| matches!(segment.status, SegmentCacheStatus::Discovered))); + } + + #[tokio::test] + async fn demand_fetch_starts_without_worker_usable_access_lease() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn demand_fetch_returns_unavailable_when_fetch_slots_are_saturated() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.active_segment_fetches = 1; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); + assert!(server.requests.lock().await.is_empty()); + } + + #[tokio::test] + async fn ready_cache_hit_is_allowed_when_fetch_slots_are_saturated() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session.active_segment_fetches = 1; + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 20 }; + } + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + assert!(server.requests.lock().await.is_empty()); + } + + #[tokio::test] + async fn origin_byterange_segment_fetch_uses_http_range_and_stores_logical_segment() { + let server = spawn_range_segment_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + origin_segment_timeout_ms: 1_000, + ..SegmentFetchPolicy::default() + }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + { + let mut session = session.write().await; + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-BYTERANGE:5@10\n#EXTINF:4.0,\n{}/big.m4s\n", + server.base_url + ))) + .expect("manifest maps"); + } + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: true, + origin_io: None, + }; + let worker = Arc::new(HlsSegmentWorkerPool::new(policy)); + let segment_file = HlsSegmentFile { proxy_seq: 0, extension: "m4s".to_string() }; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].to_ascii_lowercase().contains("range: bytes=10-14")); + let session = context.session.read().await; + let segment = session.segments.get(&0).expect("segment"); + assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 5, .. })); + assert!(context.segment_cache.metadata(&segment.cache_key).await.expect("metadata").is_some()); + } + + #[tokio::test] + async fn one_proxy_sequence_has_at_most_one_active_origin_fetch() { + let server = spawn_segment_server(80).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 4, + max_session_segment_fetches: 4, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let first = { + let worker = Arc::clone(&worker); + let context = context.clone(); + let segment_file = segment_file.clone(); + tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 20).await }) + }; + let second = { + let worker = Arc::clone(&worker); + let context = context.clone(); + let segment_file = segment_file.clone(); + tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 21).await }) + }; + + assert_eq!(first.await.expect("task"), super::SegmentDemandFetchOutcome::Ready); + assert_eq!(second.await.expect("task"), super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn retryable_407_retries_segment_fetch_until_success() { + let server = spawn_sequence_status_server(vec![407, 200]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 2); + } + + #[tokio::test] + async fn permanent_404_does_not_retry_segment_fetch() { + let server = spawn_sequence_status_server(vec![404, 200]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); + assert_eq!(server.requests.lock().await.len(), 1); + } + + #[tokio::test] + async fn segment_retry_starts_again_at_fetch_ref_after_redirect_failure() { + let server = spawn_redirect_retry_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.as_slice(), ["/1.ts", "/redirected.ts", "/1.ts"]); + } + + #[tokio::test] + async fn session_limit_is_respected() { + let server = spawn_segment_server(80).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 4, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; + + worker.wake_scheduler(context.clone(), 20).await; + tokio::time::sleep(Duration::from_millis(20)).await; + + assert_eq!(server.max_active.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn global_limit_is_respected() { + let server = spawn_segment_server(80).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 3, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; + + worker.wake_scheduler(context.clone(), 20).await; + tokio::time::sleep(Duration::from_millis(20)).await; + + assert_eq!(server.max_active.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn demand_priority_runs_before_prefetch() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session.segment_prefetch_queue = crate::api::model::SegmentPrefetchQueue::new(6); + session.segments.get_mut(&1).expect("segment").status = SegmentCacheStatus::Discovered; + session.segments.get_mut(&2).expect("segment").status = SegmentCacheStatus::Discovered; + session.queue_segment_fetch_candidate(2, SegmentFetchPriority::Prefetch, 10); + } + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); + let requests = server.requests.lock().await; + let first_request = requests.first().expect("request should be made"); + assert!(first_request.starts_with("GET /1.ts ")); + } +} diff --git a/backend/src/api/model/hls_cache/segment_repair.rs b/backend/src/api/model/hls_cache/segment_repair.rs new file mode 100644 index 000000000..fc30d8a7c --- /dev/null +++ b/backend/src/api/model/hls_cache/segment_repair.rs @@ -0,0 +1,2781 @@ +#![allow(clippy::large_futures)] + +use super::{ + safe_hls_access_lease_id, safe_proxy_session_id, segment_watchdog::HlsCorruptSegmentWatchdogManager, + CachedSegmentMetadata, HlsAccessLeaseId, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, StagedCacheObject, +}; +use crate::model::{HlsCorruptSegmentWatchdogConfig, HlsSegmentRepairConfig}; +use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; +use arc_swap::ArcSwap; +use log::{debug, warn}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + fmt, io, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{ + fs::{self, File}, + io::{AsyncRead, AsyncReadExt}, + process::Command, + sync::{Mutex, RwLock, Semaphore}, + time::timeout, +}; + +const COMMAND_VERSION: u32 = 1; +const REPAIR_METADATA_MAX_ENTRIES: usize = 4_096; +const REPAIR_OBJECT_METADATA_MAX_ENTRIES: usize = 8_192; +const REPAIR_CANDIDATE_MAX_ENTRIES: usize = 8_192; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum HlsSegmentRepairSource { + Normal, + Transient, +} + +impl HlsSegmentRepairSource { + pub(super) const fn as_log_value(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::Transient => "transient", + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum HlsRepairRenderedObjectId { + Normal { proxy_seq: u64 }, + Transient { resource_id: String }, +} + +#[derive(Debug, Clone)] +pub struct HlsSegmentRepairObjectContext { + pub source: HlsSegmentRepairSource, + pub proxy_session_id: ProxySessionId, + pub hls_access_lease_id: Option, + pub rendered_object_id: HlsRepairRenderedObjectId, + pub resource_id: String, + pub file_ext: String, + /// Concrete origin fetch URI retained for diagnostics and postprocess metadata. + /// + /// This may include a provider mirror or redirect/CDN host. It must not be used as HLS session identity, account + /// binding, provider-failover state, repair object identity, or repair-window candidate identity. + pub origin_fetch_uri_for_diagnostics: String, + pub media_sequence: Option, + pub discontinuity_sequence: Option, + pub complete_object: bool, + pub encrypted: bool, + pub custom_response: bool, +} + +impl HlsSegmentRepairObjectContext { + pub(super) fn is_repairable_ts(&self) -> bool { + self.file_ext.eq_ignore_ascii_case("ts") && self.complete_object && !self.encrypted && !self.custom_response + } + + fn repair_skip_reason(&self) -> Option<&'static str> { + if self.is_repairable_ts() { + return None; + } + if !self.file_ext.eq_ignore_ascii_case("ts") { + return Some("not-ts"); + } + if !self.complete_object { + return Some("partial-object"); + } + if self.encrypted { + return Some("encrypted"); + } + if self.custom_response { + return Some("custom-response"); + } + None + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +struct HlsRepairWindowCandidateKey { + proxy_session_id: ProxySessionId, + hls_access_lease_id: HlsAccessLeaseId, + activation_generation: u64, + /// Proxy-rendered object identity. Deliberately excludes the concrete origin fetch URI. + object_id: HlsRepairRenderedObjectId, + file_ext: String, +} + +#[derive(Debug, Clone)] +struct HlsRepairWindow { + mode: HlsSegmentRepairMode, + activation_generation: u64, + remaining_segments: u8, + seen_candidates: HashSet, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +struct RepairIdentity { + raw_sha256: String, + repair_mode: HlsSegmentRepairMode, + command_version: u32, + ffmpeg_version: String, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +struct HlsRepairObjectMetadataKey { + proxy_session_id: ProxySessionId, + /// Proxy-rendered object identity. Deliberately excludes the concrete origin fetch URI. + rendered_object_id: HlsRepairRenderedObjectId, + file_ext: String, + repair_mode: HlsSegmentRepairMode, + command_version: u32, + ffmpeg_version: String, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RepairCandidateSelection { + Selected(HlsSegmentRepairMode), + Skipped(&'static str), + AlreadyChecked, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RepairStatus { + Clean, + Fixed, + PolicyLimited, + Unsupported, + Timeout, + RemuxFailed, + ValidationFailed, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RepairVideoCodec { + H264, + Hevc, + Unsupported, +} + +impl RepairVideoCodec { + const fn as_log_value(self) -> &'static str { + match self { + Self::H264 => "h264", + Self::Hevc => "hevc", + Self::Unsupported => "unsupported", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[allow(dead_code)] +enum HlsSegmentRepairWarningKind { + MissingPat, + MissingPmt, + MissingPmtPid, + MissingVideoPidInPmt, + MissingAudioPidInPmt, + MultiplePrograms, + PesPacketSizeMismatch, + PacketCorrupt, + ContinuityCheckFailed, + MissingVps, + MissingSps, + MissingPps, + VpsOutOfRange, + SpsOutOfRange, + PpsOutOfRange, + NoFrame, + DecodeSliceHeaderError, + MissingPicture, + InvalidNal, + InvalidData, + CodecParametersMissing, + MmcoUnrefShortFailure, + ReorderBufferIncrease, + PpsIdOutOfRange, + NoStartCode, + NalSplitError, + NalParseError, + InvalidVclNalu, + InvalidMetadataNalu, + MultipleDolbyVisionRpus, + AvDesync, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +struct HlsSegmentRepairDecision { + codec: RepairVideoCodec, + required_level: HlsSegmentRepairMode, + trigger_source: HlsSegmentRepairTriggerSource, + common_low_trigger: bool, + codec_medium_trigger: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum HlsSegmentRepairTriggerSource { + Off, + CommonMpegTsLow, + H264Medium, + H264High, + HevcMedium, + HevcHigh, + UnsupportedCodec, +} + +impl HlsSegmentRepairTriggerSource { + const fn as_log_value(self) -> &'static str { + match self { + Self::Off => "off", + Self::CommonMpegTsLow => "common-mpegts-low", + Self::H264Medium => "h264-medium", + Self::H264High => "h264-high", + Self::HevcMedium => "hevc-medium", + Self::HevcHigh => "hevc-high", + Self::UnsupportedCodec => "unsupported-codec", + } + } +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct SegmentRepairMetadata { + status: RepairStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct HlsRepairObjectMetadata { + committed_sha256: String, + raw_sha256: Option, + status: RepairStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, +} + +#[derive(Debug, Clone)] +pub(super) struct HlsPostProcessingDeadline { + started: Instant, + timeout: Duration, +} + +impl HlsPostProcessingDeadline { + fn new(timeout_ms: u64) -> Self { + Self { started: Instant::now(), timeout: Duration::from_millis(timeout_ms.max(100)) } + } + + pub(super) fn remaining(&self) -> Option { self.timeout.checked_sub(self.started.elapsed()) } +} + +#[derive(Debug, Default)] +struct HlsRepairWindowRegistry { + windows: HashMap, + generations: HashMap, + checked_candidates: HashSet, + checked_candidate_order: VecDeque, +} + +impl HlsRepairWindowRegistry { + fn start_window(&mut self, lease_id: HlsAccessLeaseId, config: &HlsSegmentRepairConfig) { + if config.max_level == HlsSegmentRepairMode::Off || config.apply_to_first_segments == 0 { + self.windows.remove(&lease_id); + return; + } + let generation = self + .generations + .entry(lease_id.clone()) + .and_modify(|generation| *generation = generation.saturating_add(1)) + .or_insert(1); + self.windows.insert( + lease_id, + HlsRepairWindow { + mode: config.max_level, + activation_generation: *generation, + remaining_segments: config.apply_to_first_segments, + seen_candidates: HashSet::new(), + }, + ); + } + + fn try_select_candidate(&mut self, context: &HlsSegmentRepairObjectContext) -> RepairCandidateSelection { + if let Some(reason) = context.repair_skip_reason() { + return RepairCandidateSelection::Skipped(reason); + } + let Some(lease_id) = context.hls_access_lease_id.as_ref() else { + return RepairCandidateSelection::Skipped("missing-lease"); + }; + let Some(activation_generation) = self + .windows + .get(lease_id) + .map(|window| window.activation_generation) + .or_else(|| self.generations.get(lease_id).copied()) + else { + return RepairCandidateSelection::Skipped("no-window"); + }; + let candidate_key = HlsRepairWindowCandidateKey { + proxy_session_id: context.proxy_session_id.clone(), + hls_access_lease_id: lease_id.clone(), + activation_generation, + object_id: context.rendered_object_id.clone(), + file_ext: context.file_ext.clone(), + }; + if !self.remember_candidate(candidate_key.clone()) { + return RepairCandidateSelection::AlreadyChecked; + } + let Some(window) = self.windows.get_mut(lease_id) else { + return RepairCandidateSelection::Skipped("no-window"); + }; + if window.seen_candidates.contains(&candidate_key) { + return RepairCandidateSelection::Skipped("duplicate-identity"); + } + if window.remaining_segments == 0 { + return RepairCandidateSelection::Skipped("window-exhausted"); + } + window.seen_candidates.insert(candidate_key); + window.remaining_segments = window.remaining_segments.saturating_sub(1); + RepairCandidateSelection::Selected(window.mode) + } + + fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) { + self.windows.remove(lease_id); + self.generations.remove(lease_id); + self.checked_candidates.retain(|key| key.hls_access_lease_id != *lease_id); + self.checked_candidate_order.retain(|key| key.hls_access_lease_id != *lease_id); + } + + fn remove_proxy_session(&mut self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { + for lease_id in lease_ids { + self.windows.remove(lease_id); + self.generations.remove(lease_id); + } + self.checked_candidates.retain(|key| key.proxy_session_id != *proxy_session_id); + self.checked_candidate_order.retain(|key| key.proxy_session_id != *proxy_session_id); + } + + fn clear(&mut self) { + self.windows.clear(); + self.generations.clear(); + self.checked_candidates.clear(); + self.checked_candidate_order.clear(); + } + + fn stats(&self) -> HlsSegmentRepairStats { + HlsSegmentRepairStats { + windows: self.windows.len(), + generations: self.generations.len(), + checked_candidates: self.checked_candidates.len(), + ..HlsSegmentRepairStats::default() + } + } + + fn remember_candidate(&mut self, key: HlsRepairWindowCandidateKey) -> bool { + if !self.checked_candidates.insert(key.clone()) { + return false; + } + self.checked_candidate_order.push_back(key); + self.prune_checked_candidates(); + true + } + + fn prune_checked_candidates(&mut self) { + while self.checked_candidates.len() > REPAIR_CANDIDATE_MAX_ENTRIES { + let Some(oldest) = self.checked_candidate_order.pop_front() else { + return; + }; + self.checked_candidates.remove(&oldest); + } + } +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HlsSegmentRepairStats { + pub windows: usize, + pub generations: usize, + pub checked_candidates: usize, + pub metadata: usize, + pub object_metadata: usize, + pub locks: usize, + pub watchdog_metadata: usize, + pub watchdog_locks: usize, +} + +#[derive(Debug)] +pub struct HlsSegmentRepairManager { + runtime: ArcSwap, + watchdog: HlsCorruptSegmentWatchdogManager, + windows: RwLock, + metadata: RwLock>, + metadata_order: Mutex>, + object_metadata: RwLock>, + object_metadata_order: Mutex>, + locks: Mutex>>>, +} + +#[derive(Debug, Clone)] +struct HlsSegmentRepairRuntime { + config: HlsSegmentRepairConfig, + semaphore: Option>, + watchdog_semaphore: Arc, +} + +impl HlsSegmentRepairRuntime { + fn new(config: HlsSegmentRepairConfig) -> Self { + let semaphore = if config.max_parallel_repairs == 0 { + None + } else { + Some(Arc::new(Semaphore::new(config.max_parallel_repairs))) + }; + let watchdog_config = &config.corrupt_segment_watchdog; + let watchdog_semaphore = Arc::new(Semaphore::new(watchdog_config.max_parallel_jobs.max(1))); + Self { config, semaphore, watchdog_semaphore } + } + + fn repair_enabled(&self) -> bool { + self.config.max_level != HlsSegmentRepairMode::Off && self.config.apply_to_first_segments > 0 + } + + fn postprocessing_enabled(&self) -> bool { + self.repair_enabled() || self.config.corrupt_segment_watchdog.mode.is_enabled() + } + + fn postprocess_timeout_ms(&self) -> u64 { self.config.postprocess_timeout_ms.max(100) } +} + +fn log_segment_repair_config(config: &HlsSegmentRepairConfig) { + debug!( + "HLS segment repair configured: max_level={} segments={} max_parallel={} postprocess_timeout_ms={}", + config.max_level.as_log_value(), + config.apply_to_first_segments, + config.max_parallel_repairs, + config.postprocess_timeout_ms + ); +} + +fn log_corrupt_segment_watchdog_config(config: &HlsCorruptSegmentWatchdogConfig) { + debug!( + "HLS corrupt segment watchdog configured: mode={} max_parallel_jobs={}", + config.mode.as_log_value(), + config.max_parallel_jobs + ); +} + +impl HlsSegmentRepairManager { + pub fn new(config: HlsSegmentRepairConfig) -> Self { + let watchdog_config = config.corrupt_segment_watchdog.clone(); + log_segment_repair_config(&config); + log_corrupt_segment_watchdog_config(&watchdog_config); + Self { + runtime: ArcSwap::from_pointee(HlsSegmentRepairRuntime::new(config)), + watchdog: HlsCorruptSegmentWatchdogManager::new(), + windows: RwLock::new(HlsRepairWindowRegistry::default()), + metadata: RwLock::new(HashMap::new()), + metadata_order: Mutex::new(VecDeque::new()), + object_metadata: RwLock::new(HashMap::new()), + object_metadata_order: Mutex::new(VecDeque::new()), + locks: Mutex::new(HashMap::new()), + } + } + + pub fn update_config(&self, config: HlsSegmentRepairConfig) { + let current = self.runtime.load(); + if current.config == config { + return; + } + let watchdog_config = config.corrupt_segment_watchdog.clone(); + log_segment_repair_config(&config); + log_corrupt_segment_watchdog_config(&watchdog_config); + self.runtime.store(Arc::new(HlsSegmentRepairRuntime::new(config))); + } + + pub async fn start_access_lease_window(&self, lease_id: HlsAccessLeaseId) { + let runtime = self.runtime.load_full(); + if !runtime.repair_enabled() { + return; + } + self.windows.write().await.start_window(lease_id.clone(), &runtime.config); + debug!( + "HLS segment repair window started: lease={} max_level={} segments={}", + safe_hls_access_lease_id(&lease_id), + runtime.config.max_level.as_log_value(), + runtime.config.apply_to_first_segments + ); + } + + pub async fn remove_access_lease_window(&self, lease_id: &HlsAccessLeaseId) { + self.windows.write().await.remove_access_lease(lease_id); + } + + pub async fn remove_proxy_session_state(&self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { + self.windows.write().await.remove_proxy_session(proxy_session_id, lease_ids); + self.object_metadata.write().await.retain(|key, _| key.proxy_session_id != *proxy_session_id); + self.object_metadata_order.lock().await.retain(|key| key.proxy_session_id != *proxy_session_id); + } + + pub async fn clear_runtime_state(&self) { + self.windows.write().await.clear(); + self.metadata.write().await.clear(); + self.metadata_order.lock().await.clear(); + self.object_metadata.write().await.clear(); + self.object_metadata_order.lock().await.clear(); + self.locks.lock().await.clear(); + self.watchdog.clear_runtime_state().await; + } + + pub async fn stats(&self) -> HlsSegmentRepairStats { + let mut stats = self.windows.read().await.stats(); + stats.metadata = self.metadata.read().await.len(); + stats.object_metadata = self.object_metadata.read().await.len(); + stats.locks = self.locks.lock().await.len(); + let watchdog = self.watchdog.stats().await; + stats.watchdog_metadata = watchdog.metadata; + stats.watchdog_locks = watchdog.locks; + stats + } + + pub async fn commit_origin_response( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + reader: R, + deadline: Duration, + context: HlsSegmentRepairObjectContext, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let raw = segment_cache.stage_temp_with_timeout(key, reader, deadline).await?; + let runtime = self.runtime.load_full(); + if !runtime.postprocessing_enabled() { + return segment_cache.commit_staged(key, raw).await; + } + let postprocessing_deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); + self.process_staged_and_commit(segment_cache, key, raw, context, runtime, postprocessing_deadline).await + } + + #[allow(clippy::too_many_lines)] + pub async fn repair_ready_cache_hit( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + context: HlsSegmentRepairObjectContext, + ) -> io::Result> + where + K: HlsCacheObjectKey, + { + if !self.runtime.load().repair_enabled() { + return Ok(None); + } + let Some((mode, runtime)) = self.try_select_candidate(&context).await else { + return Ok(None); + }; + let Some(metadata) = segment_cache.metadata(key).await? else { + return Ok(None); + }; + let raw_hash = sha256_file(&metadata.path).await?; + let object_key = repair_object_metadata_key(&context, mode); + if self.object_metadata_matches(&object_key, &raw_hash).await { + return Ok(None); + } + let identity = RepairIdentity { + raw_sha256: raw_hash.clone(), + repair_mode: mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + }; + if self.repair_metadata(&identity).await.is_some() { + self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) + .await; + return Ok(None); + } + let lock = self.lock_for_identity(identity.clone()).await; + let result = { + let _guard = lock.lock().await; + if let Some(current_metadata) = segment_cache.metadata(key).await? { + let current_hash = sha256_file(¤t_metadata.path).await?; + if current_hash != raw_hash { + Ok(None) + } else if self.repair_metadata(&identity).await.is_some() { + self.record_object_metadata_from_repair_identity( + object_key.clone(), + current_hash.clone(), + Some(current_hash), + &identity, + ) + .await; + Ok(None) + } else { + let deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); + if let Some(fixed_path) = self + .repair_file( + ¤t_metadata.path, + current_metadata.size, + &identity, + &context, + runtime.clone(), + &deadline, + ) + .await? + { + let fixed_size = fs::metadata(&fixed_path).await?.len(); + let staged = StagedCacheObject { path: fixed_path, size: fixed_size }; + let committed = segment_cache.commit_staged(key, staged).await?; + self.record_metadata( + identity.clone(), + RepairStatus::Fixed, + current_metadata.size, + committed.size, + None, + ) + .await; + let committed_hash = sha256_file(&committed.path).await?; + self.record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: committed_hash, + raw_sha256: Some(current_hash), + status: RepairStatus::Fixed, + raw_size: current_metadata.size, + final_size: committed.size, + validation_reason: None, + }, + ) + .await; + Ok(Some(committed)) + } else { + self.record_object_metadata_from_repair_identity( + object_key.clone(), + current_hash.clone(), + Some(current_hash), + &identity, + ) + .await; + Ok(None) + } + } + } else { + Ok(None) + } + }; + self.remove_lock_if_unused(&identity, &lock).await; + result + } + + async fn process_staged_and_commit( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + context: HlsSegmentRepairObjectContext, + runtime: Arc, + deadline: HlsPostProcessingDeadline, + ) -> io::Result + where + K: HlsCacheObjectKey, + { + let selected_repair = self.try_select_candidate_with_runtime(&context, &runtime).await; + if selected_repair.is_none() + && runtime.config.corrupt_segment_watchdog.mode.is_enabled() + && context.is_repairable_ts() + { + let raw_hash = sha256_file(&raw.path).await?; + return self + .watchdog + .process_staged_and_commit( + segment_cache, + key, + raw, + &context, + &runtime.config.corrupt_segment_watchdog, + &runtime.watchdog_semaphore, + raw_hash, + &deadline, + ) + .await; + } + let Some(mode) = selected_repair else { + return segment_cache.commit_staged(key, raw).await; + }; + let raw_hash = sha256_file(&raw.path).await?; + let object_key = repair_object_metadata_key(&context, mode); + if self.object_metadata_matches(&object_key, &raw_hash).await { + return segment_cache.commit_staged(key, raw).await; + } + let identity = RepairIdentity { + raw_sha256: raw_hash.clone(), + repair_mode: mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + }; + if self.repair_metadata(&identity).await.is_some() { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) + .await; + return Ok(committed); + } + let lock = self.lock_for_identity(identity.clone()).await; + let result = { + let _guard = lock.lock().await; + if self.repair_metadata(&identity).await.is_some() { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity( + object_key.clone(), + raw_hash.clone(), + Some(raw_hash.clone()), + &identity, + ) + .await; + Ok(committed) + } else { + let raw_size = raw.size; + if let Some(fixed_path) = + self.repair_file(&raw.path, raw.size, &identity, &context, runtime.clone(), &deadline).await? + { + let fixed_size = fs::metadata(&fixed_path).await?.len(); + let _ = segment_cache.remove_staged(raw.clone()).await; + let committed = segment_cache + .commit_staged(key, StagedCacheObject { path: fixed_path, size: fixed_size }) + .await?; + self.record_metadata(identity.clone(), RepairStatus::Fixed, raw.size, committed.size, None).await; + let committed_hash = sha256_file(&committed.path).await?; + self.record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: committed_hash, + raw_sha256: Some(raw_hash.clone()), + status: RepairStatus::Fixed, + raw_size, + final_size: committed.size, + validation_reason: None, + }, + ) + .await; + Ok(committed) + } else { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity( + object_key.clone(), + raw_hash.clone(), + Some(raw_hash.clone()), + &identity, + ) + .await; + Ok(committed) + } + } + }; + self.remove_lock_if_unused(&identity, &lock).await; + result + } + + async fn try_select_candidate( + &self, + context: &HlsSegmentRepairObjectContext, + ) -> Option<(HlsSegmentRepairMode, Arc)> { + let runtime = self.runtime.load_full(); + self.try_select_candidate_with_runtime(context, &runtime).await.map(|mode| (mode, runtime)) + } + + async fn try_select_candidate_with_runtime( + &self, + context: &HlsSegmentRepairObjectContext, + runtime: &Arc, + ) -> Option { + if !runtime.repair_enabled() { + return None; + } + context.hls_access_lease_id.as_ref()?; + match self.windows.write().await.try_select_candidate(context) { + RepairCandidateSelection::Selected(mode) => { + debug!( + "HLS segment repair candidate selected: session={} lease={} source={} resource={} mode={}", + safe_proxy_session_id(&context.proxy_session_id), + context.hls_access_lease_id.as_ref().map_or_else(|| "".to_string(), safe_hls_access_lease_id), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ); + Some(mode) + } + RepairCandidateSelection::Skipped(reason) => { + debug!( + "HLS segment repair candidate skipped: session={} lease={} source={} resource={} reason={}", + safe_proxy_session_id(&context.proxy_session_id), + context.hls_access_lease_id.as_ref().map_or_else(|| "".to_string(), safe_hls_access_lease_id), + context.source.as_log_value(), + context.resource_id, + reason + ); + None + } + RepairCandidateSelection::AlreadyChecked => None, + } + } + + #[allow(clippy::too_many_lines)] + async fn repair_file( + &self, + raw_path: &Path, + raw_size: u64, + identity: &RepairIdentity, + context: &HlsSegmentRepairObjectContext, + runtime: Arc, + deadline: &HlsPostProcessingDeadline, + ) -> io::Result> { + let _permit = match &runtime.semaphore { + Some(semaphore) => { + let Some(remaining) = deadline.remaining() else { + self.record_metadata( + identity.clone(), + RepairStatus::Timeout, + raw_size, + raw_size, + Some("timeout".to_string()), + ) + .await; + return Ok(None); + }; + Some( + timeout(remaining, semaphore.acquire()) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "repair semaphore timed out"))? + .map_err(|_| io::Error::other("repair semaphore closed"))?, + ) + } + None => None, + }; + let started = Instant::now(); + let raw_scan = match analyze_segment(raw_path, deadline).await { + Ok(scan) => scan, + Err(reason) => { + debug_repair_event(context, identity.repair_mode, "analysis skipped", Some(&reason)); + self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + }; + let codec = detect_video_codec(&raw_scan); + let decision = decide_repair(codec, &raw_scan.warnings); + let execution_plan = identity.repair_mode.execution_plan(decision.required_level); + let executed_level = match execution_plan { + HlsSegmentRepairExecutionPlan::Repair(level) => Some(level), + HlsSegmentRepairExecutionPlan::SkipNoTrigger + | HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => None, + }; + debug_repair_analysis(context, identity.repair_mode, decision, executed_level, &raw_scan.warnings); + let executed_level = match execution_plan { + HlsSegmentRepairExecutionPlan::Repair(level) => level, + HlsSegmentRepairExecutionPlan::SkipNoTrigger => { + self.record_metadata(identity.clone(), RepairStatus::Clean, raw_size, raw_size, None).await; + return Ok(None); + } + HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => { + warn!( + "HLS segment repair required level exceeds configured max: session={} source={} resource={} configured_max_level={} required_level={} trigger={} action=raw_commit", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + decision.required_level.as_log_value(), + decision.trigger_source.as_log_value() + ); + self.record_metadata( + identity.clone(), + RepairStatus::PolicyLimited, + raw_size, + raw_size, + Some("configured_max_level_below_required_level".to_string()), + ) + .await; + return Ok(None); + } + }; + let stream_selection = match select_repair_remux_streams(&raw_scan) { + Ok(selection) => selection, + Err(reason) => { + debug_repair_event(context, executed_level, "stream selection skipped", Some(&reason)); + self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + }; + for dropped in &stream_selection.dropped_streams { + debug_repair_stream_dropped(context, dropped); + } + let fixed_path = repair_output_path(raw_path); + debug!( + "HLS segment repair remux started: session={} source={} resource={} configured_max_level={} executed_level={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + executed_level.as_log_value() + ); + let remux = run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline).await; + if let Err(reason) = remux { + debug_repair_event(context, executed_level, "remux failed", Some(&reason)); + let status = if reason == "timeout" { RepairStatus::Timeout } else { RepairStatus::RemuxFailed }; + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata(identity.clone(), status, raw_size, raw_size, Some(reason)).await; + return Ok(None); + } + let fixed_scan = match analyze_segment(&fixed_path, deadline).await { + Ok(scan) => scan, + Err(reason) => { + debug_repair_event(context, executed_level, "validation probe failed", Some(&reason)); + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata( + identity.clone(), + RepairStatus::ValidationFailed, + raw_size, + raw_size, + Some(reason), + ) + .await; + return Ok(None); + } + }; + let validation = validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); + if let Err(reason) = validation { + debug_repair_event(context, executed_level, "validation failed", Some(&reason)); + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata(identity.clone(), RepairStatus::ValidationFailed, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + debug!( + "HLS segment repair remux completed: session={} source={} resource={} configured_max_level={} executed_level={} duration_ms={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + executed_level.as_log_value(), + started.elapsed().as_millis() + ); + Ok(Some(fixed_path)) + } + + async fn record_metadata( + &self, + identity: RepairIdentity, + status: RepairStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, + ) { + let inserted_new = { + let mut metadata = self.metadata.write().await; + let inserted_new = !metadata.contains_key(&identity); + metadata + .insert(identity.clone(), SegmentRepairMetadata { status, raw_size, final_size, validation_reason }); + inserted_new + }; + if inserted_new { + self.metadata_order.lock().await.push_back(identity); + } + self.prune_metadata().await; + } + + async fn repair_metadata(&self, identity: &RepairIdentity) -> Option { + self.metadata.read().await.get(identity).cloned() + } + + async fn object_metadata_matches(&self, key: &HlsRepairObjectMetadataKey, committed_sha256: &str) -> bool { + let matches = self + .object_metadata + .read() + .await + .get(key) + .is_some_and(|metadata| metadata.committed_sha256 == committed_sha256); + if matches { + let (source, resource) = match &key.rendered_object_id { + HlsRepairRenderedObjectId::Normal { proxy_seq } => ("normal", format!("{proxy_seq:06}")), + HlsRepairRenderedObjectId::Transient { resource_id } => ("transient", resource_id.clone()), + }; + debug!( + "HLS segment repair object metadata hit: session={} source={} resource={} mode={}", + safe_proxy_session_id(&key.proxy_session_id), + source, + resource, + key.repair_mode.as_log_value() + ); + } + matches + } + + async fn record_object_metadata_from_repair_identity( + &self, + key: HlsRepairObjectMetadataKey, + committed_sha256: String, + raw_sha256: Option, + identity: &RepairIdentity, + ) { + let Some(metadata) = self.repair_metadata(identity).await else { + return; + }; + self.record_object_metadata( + key, + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256, + status: metadata.status, + raw_size: metadata.raw_size, + final_size: metadata.final_size, + validation_reason: metadata.validation_reason, + }, + ) + .await; + } + + async fn record_object_metadata(&self, key: HlsRepairObjectMetadataKey, metadata: HlsRepairObjectMetadata) { + let inserted_new = { + let mut object_metadata = self.object_metadata.write().await; + let inserted_new = !object_metadata.contains_key(&key); + object_metadata.insert(key.clone(), metadata); + inserted_new + }; + if inserted_new { + self.object_metadata_order.lock().await.push_back(key); + } + self.prune_object_metadata().await; + } + + async fn lock_for_identity(&self, identity: RepairIdentity) -> Arc> { + let mut locks = self.locks.lock().await; + Arc::clone(locks.entry(identity).or_insert_with(|| Arc::new(Mutex::new(())))) + } + + async fn remove_lock_if_unused(&self, identity: &RepairIdentity, lock: &Arc>) { + let mut locks = self.locks.lock().await; + if Arc::strong_count(lock) <= 2 && locks.get(identity).is_some_and(|current| Arc::ptr_eq(current, lock)) { + locks.remove(identity); + } + } + + async fn prune_metadata(&self) { + loop { + let should_prune = self.metadata.read().await.len() > REPAIR_METADATA_MAX_ENTRIES; + if !should_prune { + return; + } + let Some(oldest) = self.metadata_order.lock().await.pop_front() else { + return; + }; + self.metadata.write().await.remove(&oldest); + } + } + + async fn prune_object_metadata(&self) { + loop { + let should_prune = self.object_metadata.read().await.len() > REPAIR_OBJECT_METADATA_MAX_ENTRIES; + if !should_prune { + return; + } + let Some(oldest) = self.object_metadata_order.lock().await.pop_front() else { + return; + }; + self.object_metadata.write().await.remove(&oldest); + } + } +} + +fn repair_object_metadata_key( + context: &HlsSegmentRepairObjectContext, + repair_mode: HlsSegmentRepairMode, +) -> HlsRepairObjectMetadataKey { + HlsRepairObjectMetadataKey { + proxy_session_id: context.proxy_session_id.clone(), + rendered_object_id: context.rendered_object_id.clone(), + file_ext: context.file_ext.to_ascii_lowercase(), + repair_mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + } +} + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct WarningCounters { + pub missing_pat: u32, + pub missing_pmt: u32, + pub missing_pmt_pid: u32, + pub missing_video_pid_in_pmt: u32, + pub missing_audio_pid_in_pmt: u32, + pub multiple_programs: u32, + pub pes_packet_size_mismatch: u32, + pub packet_corrupt: u32, + pub continuity_check_failed: u32, + pub missing_vps: u32, + pub missing_sps: u32, + pub missing_pps: u32, + pub vps_out_of_range: u32, + pub sps_out_of_range: u32, + pub pps_out_of_range: u32, + pub no_frame: u32, + pub decode_slice_header_error: u32, + pub missing_picture: u32, + pub invalid_nal: u32, + pub invalid_data: u32, + pub codec_parameters_missing: u32, + pub mmco_unref_short_failure: u32, + pub reorder_buffer: u32, + pub pps_id_out_of_range: u32, + pub no_start_code: u32, + pub nal_split_error: u32, + pub nal_parse_error: u32, + pub invalid_undecodable_nalu_total: u32, + pub invalid_undecodable_nalu_non_metadata: u32, + pub invalid_undecodable_nalu_keyframe: u32, + pub invalid_undecodable_nalu_metadata: u32, + pub dolby_vision_rpu: u32, + pub av_desync: u32, +} + +#[derive(Debug, Clone, Default)] +struct SegmentProbe { + duration_ms: Option, + size: u64, + stream_count: usize, + streams: Vec, + primary_video_codec: Option, + primary_audio_codec: Option, + primary_video_start_time_ms: Option, + primary_audio_start_time_ms: Option, + primary_video_extradata_size: Option, + warnings: WarningCounters, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct SegmentProbeStream { + index: usize, + stream_type: SegmentProbeStreamType, + codec_name: Option, + width: Option, + height: Option, + sample_rate: Option, + channels: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum SegmentProbeStreamType { + Video, + Audio, + Other, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct RepairRemuxStreamSelection { + mapped_streams: Vec, + dropped_streams: Vec, +} + +#[cfg(test)] +impl RepairRemuxStreamSelection { + fn preserve_all(probe: &SegmentProbe) -> Self { + Self { + mapped_streams: probe.streams.iter().map(|stream| stream.index).collect(), + dropped_streams: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct RepairRemuxDroppedStream { + index: usize, + reason: &'static str, +} + +pub fn parse_ffmpeg_warnings(stderr: &str) -> WarningCounters { + let mut counters = WarningCounters::default(); + let mut last_increment: Option = None; + for line in stderr.lines() { + let trimmed = line.trim(); + if let Some(repeated) = parse_repeated_count(trimmed) { + if let Some(increment) = last_increment { + increment(&mut counters, repeated); + } + continue; + } + if let Some(increment) = warning_increment(trimmed) { + increment(&mut counters, 1); + last_increment = Some(increment); + } + } + counters +} + +fn parse_repeated_count(line: &str) -> Option { + let rest = line.strip_prefix("Last message repeated ")?; + let count = rest.strip_suffix(" times")?.parse().ok()?; + Some(count) +} + +#[allow(clippy::too_many_lines)] +fn warning_increment(line: &str) -> Option { + let lower = line.to_ascii_lowercase(); + if lower.contains("missing pat") { + return Some(|counters, count| counters.missing_pat = counters.missing_pat.saturating_add(count)); + } + if lower.contains("missing pmt pid") { + return Some(|counters, count| counters.missing_pmt_pid = counters.missing_pmt_pid.saturating_add(count)); + } + if lower.contains("missing pmt") { + return Some(|counters, count| counters.missing_pmt = counters.missing_pmt.saturating_add(count)); + } + if lower.contains("missing video pid") && lower.contains("pmt") { + return Some(|counters, count| { + counters.missing_video_pid_in_pmt = counters.missing_video_pid_in_pmt.saturating_add(count); + }); + } + if lower.contains("missing audio pid") && lower.contains("pmt") { + return Some(|counters, count| { + counters.missing_audio_pid_in_pmt = counters.missing_audio_pid_in_pmt.saturating_add(count); + }); + } + if lower.contains("multiple mpeg-ts programs") || lower.contains("multiple programs") { + return Some(|counters, count| counters.multiple_programs = counters.multiple_programs.saturating_add(count)); + } + if lower.contains("pes packet size mismatch") { + return Some(|counters, count| { + counters.pes_packet_size_mismatch = counters.pes_packet_size_mismatch.saturating_add(count); + }); + } + if lower.contains("packet corrupt") { + return Some(|counters, count| counters.packet_corrupt = counters.packet_corrupt.saturating_add(count)); + } + if lower.contains("continuity check failed") { + return Some(|counters, count| { + counters.continuity_check_failed = counters.continuity_check_failed.saturating_add(count); + }); + } + if lower.contains("non-existing vps") || lower.contains("missing vps") { + return Some(|counters, count| counters.missing_vps = counters.missing_vps.saturating_add(count)); + } + if lower.contains("non-existing sps") || lower.contains("missing sps") { + return Some(|counters, count| counters.missing_sps = counters.missing_sps.saturating_add(count)); + } + if lower.contains("non-existing pps") || lower.contains("missing pps") { + return Some(|counters, count| counters.missing_pps = counters.missing_pps.saturating_add(count)); + } + if lower.contains("vps id out of range") { + return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); + } + if lower.contains("sps id out of range") { + return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); + } + if lower.contains("pps id out of range") { + return Some(|counters, count| { + counters.pps_id_out_of_range = counters.pps_id_out_of_range.saturating_add(count); + counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count); + }); + } + if lower.contains("vps") && lower.contains("out of range") { + return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); + } + if lower.contains("sps") && lower.contains("out of range") { + return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); + } + if lower.contains("pps") && lower.contains("out of range") { + return Some(|counters, count| counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count)); + } + if lower.contains("no frame") { + return Some(|counters, count| counters.no_frame = counters.no_frame.saturating_add(count)); + } + if lower.contains("decode_slice_header error") { + return Some(|counters, count| { + counters.decode_slice_header_error = counters.decode_slice_header_error.saturating_add(count); + }); + } + if lower.contains("missing picture") { + return Some(|counters, count| counters.missing_picture = counters.missing_picture.saturating_add(count)); + } + if lower.contains("invalid nal") { + return Some(|counters, count| counters.invalid_nal = counters.invalid_nal.saturating_add(count)); + } + if lower.contains("invalid data found") { + return Some(|counters, count| counters.invalid_data = counters.invalid_data.saturating_add(count)); + } + if lower.contains("could not find codec parameters") { + return Some(|counters, count| { + counters.codec_parameters_missing = counters.codec_parameters_missing.saturating_add(count); + }); + } + if lower.contains("mmco: unref short failure") { + return Some(|counters, count| { + counters.mmco_unref_short_failure = counters.mmco_unref_short_failure.saturating_add(count); + }); + } + if lower.contains("increasing reorder buffer") { + return Some(|counters, count| counters.reorder_buffer = counters.reorder_buffer.saturating_add(count)); + } + if lower.contains("no start code is found") { + return Some(|counters, count| counters.no_start_code = counters.no_start_code.saturating_add(count)); + } + if lower.contains("error splitting") && lower.contains("nal") { + return Some(|counters, count| counters.nal_split_error = counters.nal_split_error.saturating_add(count)); + } + if lower.contains("error parsing nal unit") { + return Some(|counters, count| counters.nal_parse_error = counters.nal_parse_error.saturating_add(count)); + } + if let Some(nalu_type) = parse_invalid_undecodable_nalu_type(line) { + return Some(match nalu_type { + 0..=20 | 22..=31 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_non_metadata = + counters.invalid_undecodable_nalu_non_metadata.saturating_add(count); + }, + 21 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_keyframe = + counters.invalid_undecodable_nalu_keyframe.saturating_add(count); + }, + 39 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_metadata = + counters.invalid_undecodable_nalu_metadata.saturating_add(count); + }, + _ => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + }, + }); + } + if lower.contains("multiple dolby vision rpus found in one au") { + return Some(|counters, count| counters.dolby_vision_rpu = counters.dolby_vision_rpu.saturating_add(count)); + } + if lower.contains("audio/video desynchronisation detected") { + return Some(|counters, count| counters.av_desync = counters.av_desync.saturating_add(count)); + } + None +} + +fn parse_invalid_undecodable_nalu_type(line: &str) -> Option { + let (_, rest) = line.split_once("Skipping invalid undecodable NALU:")?; + rest.trim().split(|ch: char| !ch.is_ascii_digit()).next().filter(|value| !value.is_empty())?.parse().ok() +} + +fn warning_count(warnings: &WarningCounters, kind: HlsSegmentRepairWarningKind) -> u32 { + match kind { + HlsSegmentRepairWarningKind::MissingPat => warnings.missing_pat, + HlsSegmentRepairWarningKind::MissingPmt => warnings.missing_pmt, + HlsSegmentRepairWarningKind::MissingPmtPid => warnings.missing_pmt_pid, + HlsSegmentRepairWarningKind::MissingVideoPidInPmt => warnings.missing_video_pid_in_pmt, + HlsSegmentRepairWarningKind::MissingAudioPidInPmt => warnings.missing_audio_pid_in_pmt, + HlsSegmentRepairWarningKind::MultiplePrograms => warnings.multiple_programs, + HlsSegmentRepairWarningKind::PesPacketSizeMismatch => warnings.pes_packet_size_mismatch, + HlsSegmentRepairWarningKind::PacketCorrupt => warnings.packet_corrupt, + HlsSegmentRepairWarningKind::ContinuityCheckFailed => warnings.continuity_check_failed, + HlsSegmentRepairWarningKind::MissingVps => warnings.missing_vps, + HlsSegmentRepairWarningKind::MissingSps => warnings.missing_sps, + HlsSegmentRepairWarningKind::MissingPps => warnings.missing_pps, + HlsSegmentRepairWarningKind::VpsOutOfRange => warnings.vps_out_of_range, + HlsSegmentRepairWarningKind::SpsOutOfRange => warnings.sps_out_of_range, + HlsSegmentRepairWarningKind::PpsOutOfRange => warnings.pps_out_of_range, + HlsSegmentRepairWarningKind::NoFrame => warnings.no_frame, + HlsSegmentRepairWarningKind::DecodeSliceHeaderError => warnings.decode_slice_header_error, + HlsSegmentRepairWarningKind::MissingPicture => warnings.missing_picture, + HlsSegmentRepairWarningKind::InvalidNal => warnings.invalid_nal, + HlsSegmentRepairWarningKind::InvalidData => warnings.invalid_data, + HlsSegmentRepairWarningKind::CodecParametersMissing => warnings.codec_parameters_missing, + HlsSegmentRepairWarningKind::MmcoUnrefShortFailure => warnings.mmco_unref_short_failure, + HlsSegmentRepairWarningKind::ReorderBufferIncrease => warnings.reorder_buffer, + HlsSegmentRepairWarningKind::PpsIdOutOfRange => warnings.pps_id_out_of_range, + HlsSegmentRepairWarningKind::NoStartCode => warnings.no_start_code, + HlsSegmentRepairWarningKind::NalSplitError => warnings.nal_split_error, + HlsSegmentRepairWarningKind::NalParseError => warnings.nal_parse_error, + HlsSegmentRepairWarningKind::InvalidVclNalu => { + warnings.invalid_undecodable_nalu_non_metadata.saturating_add(warnings.invalid_undecodable_nalu_keyframe) + } + HlsSegmentRepairWarningKind::InvalidMetadataNalu => warnings.invalid_undecodable_nalu_metadata, + HlsSegmentRepairWarningKind::MultipleDolbyVisionRpus => warnings.dolby_vision_rpu, + HlsSegmentRepairWarningKind::AvDesync => warnings.av_desync, + } +} + +fn has_any_warning(warnings: &WarningCounters, kinds: &[HlsSegmentRepairWarningKind]) -> bool { + kinds.iter().any(|kind| warning_count(warnings, *kind) > 0) +} + +fn common_mpegts_low_trigger(warnings: &WarningCounters) -> bool { + has_any_warning( + warnings, + &[ + HlsSegmentRepairWarningKind::MissingPat, + HlsSegmentRepairWarningKind::MissingPmt, + HlsSegmentRepairWarningKind::MissingPmtPid, + HlsSegmentRepairWarningKind::MissingVideoPidInPmt, + HlsSegmentRepairWarningKind::MissingAudioPidInPmt, + HlsSegmentRepairWarningKind::MultiplePrograms, + HlsSegmentRepairWarningKind::PesPacketSizeMismatch, + HlsSegmentRepairWarningKind::PacketCorrupt, + HlsSegmentRepairWarningKind::ContinuityCheckFailed, + HlsSegmentRepairWarningKind::CodecParametersMissing, + ], + ) +} + +fn h264_parameter_trigger(warnings: &WarningCounters) -> bool { + has_any_warning( + warnings, + &[ + HlsSegmentRepairWarningKind::MissingSps, + HlsSegmentRepairWarningKind::MissingPps, + HlsSegmentRepairWarningKind::SpsOutOfRange, + HlsSegmentRepairWarningKind::PpsOutOfRange, + ], + ) +} + +fn h264_medium_trigger(warnings: &WarningCounters) -> bool { + h264_parameter_trigger(warnings) + || warnings.decode_slice_header_error > 0 + || warnings.no_frame > 0 + || (warnings.missing_picture > 0 + && (warnings.missing_sps > 0 || warnings.missing_pps > 0 || warnings.decode_slice_header_error > 0)) + || (warnings.invalid_nal > 0 && h264_parameter_trigger(warnings)) + || (warnings.no_start_code > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) + || (warnings.nal_split_error > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) +} + +fn hevc_missing_parameter_trigger(warnings: &WarningCounters) -> bool { + warnings.missing_vps > 0 || warnings.missing_sps > 0 || warnings.missing_pps > 0 +} + +fn hevc_parameter_trigger(warnings: &WarningCounters) -> bool { + hevc_missing_parameter_trigger(warnings) + || warnings.vps_out_of_range > 0 + || warnings.sps_out_of_range > 0 + || warnings.pps_out_of_range > 0 + || warnings.pps_id_out_of_range > 0 +} + +fn hevc_invalid_vcl_nalu(warnings: &WarningCounters) -> bool { + warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu) > 0 +} + +fn hevc_medium_trigger(warnings: &WarningCounters) -> bool { + hevc_parameter_trigger(warnings) + || (hevc_invalid_vcl_nalu(warnings) && hevc_parameter_trigger(warnings)) + || (warnings.nal_parse_error > 0 && hevc_parameter_trigger(warnings)) + || (warnings.invalid_nal > 0 && hevc_parameter_trigger(warnings)) + || (warnings.no_start_code > 0 && hevc_missing_parameter_trigger(warnings)) + || (warnings.nal_split_error > 0 && hevc_missing_parameter_trigger(warnings)) +} + +fn decide_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> HlsSegmentRepairDecision { + let common_low_trigger = common_mpegts_low_trigger(warnings); + let codec_parameters_missing = warnings.codec_parameters_missing > 0; + let codec_medium_trigger = match codec { + RepairVideoCodec::H264 => h264_medium_trigger(warnings), + RepairVideoCodec::Hevc => hevc_medium_trigger(warnings), + RepairVideoCodec::Unsupported => false, + }; + match codec { + RepairVideoCodec::H264 if codec_medium_trigger && (common_low_trigger || codec_parameters_missing) => { + HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::High, + trigger_source: HlsSegmentRepairTriggerSource::H264High, + common_low_trigger, + codec_medium_trigger, + } + } + RepairVideoCodec::H264 if codec_medium_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Medium, + trigger_source: HlsSegmentRepairTriggerSource::H264Medium, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::Hevc + if (common_low_trigger && (codec_medium_trigger || hevc_invalid_vcl_nalu(warnings))) + || (codec_parameters_missing && codec_medium_trigger) => + { + HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::High, + trigger_source: HlsSegmentRepairTriggerSource::HevcHigh, + common_low_trigger, + codec_medium_trigger, + } + } + RepairVideoCodec::Hevc if codec_medium_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Medium, + trigger_source: HlsSegmentRepairTriggerSource::HevcMedium, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::H264 | RepairVideoCodec::Hevc if common_low_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Low, + trigger_source: HlsSegmentRepairTriggerSource::CommonMpegTsLow, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::Unsupported => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Off, + trigger_source: HlsSegmentRepairTriggerSource::UnsupportedCodec, + common_low_trigger, + codec_medium_trigger, + }, + _ => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Off, + trigger_source: HlsSegmentRepairTriggerSource::Off, + common_low_trigger, + codec_medium_trigger, + }, + } +} + +fn debug_repair_analysis( + context: &HlsSegmentRepairObjectContext, + configured_max_level: HlsSegmentRepairMode, + decision: HlsSegmentRepairDecision, + executed_level: Option, + warnings: &WarningCounters, +) { + let executed_level = executed_level.map_or("off", HlsSegmentRepairMode::as_log_value); + match decision.codec { + RepairVideoCodec::H264 => debug!( + "HLS segment repair analysis completed: session={} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_sps={} missing_pps={} sps_out_of_range={} pps_out_of_range={} no_frame={} decode_slice_header_error={} missing_picture={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value(), + decision.common_low_trigger, + decision.codec_medium_trigger, + warnings.missing_sps, + warnings.missing_pps, + warnings.sps_out_of_range, + warnings.pps_out_of_range, + warnings.no_frame, + warnings.decode_slice_header_error, + warnings.missing_picture, + warnings.invalid_nal, + warnings.no_start_code, + warnings.nal_split_error, + warnings.packet_corrupt, + warnings.continuity_check_failed, + warnings.codec_parameters_missing + ), + RepairVideoCodec::Hevc => debug!( + "HLS segment repair analysis completed: session={} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_vps={} missing_sps={} missing_pps={} vps_out_of_range={} sps_out_of_range={} pps_out_of_range={} pps_id_out_of_range={} invalid_vcl_nalu={} nal_parse_error={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value(), + decision.common_low_trigger, + decision.codec_medium_trigger, + warnings.missing_vps, + warnings.missing_sps, + warnings.missing_pps, + warnings.vps_out_of_range, + warnings.sps_out_of_range, + warnings.pps_out_of_range, + warnings.pps_id_out_of_range, + warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu), + warnings.nal_parse_error, + warnings.invalid_nal, + warnings.no_start_code, + warnings.nal_split_error, + warnings.packet_corrupt, + warnings.continuity_check_failed, + warnings.codec_parameters_missing + ), + RepairVideoCodec::Unsupported => debug!( + "HLS segment repair analysis completed: session={} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value() + ), + } +} + +fn debug_repair_event( + context: &HlsSegmentRepairObjectContext, + mode: HlsSegmentRepairMode, + event: &'static str, + reason: Option<&str>, +) { + if let Some(reason) = reason { + debug!( + "HLS segment repair {event}: session={} source={} resource={} mode={} reason={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value(), + reason + ); + } else { + debug!( + "HLS segment repair {event}: session={} source={} resource={} mode={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ); + } +} + +fn debug_repair_stream_dropped(context: &HlsSegmentRepairObjectContext, dropped: &RepairRemuxDroppedStream) { + debug!( + "HLS segment repair stream dropped: session={} source={} resource={} stream={} reason={}", + safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + dropped.index, + dropped.reason + ); +} + +async fn analyze_segment(path: &Path, deadline: &HlsPostProcessingDeadline) -> Result { + let probe_output = run_command_with_deadline( + "ffprobe", + &[ + "-hide_banner", + "-v", + "warning", + "-show_entries", + "format=duration,size,bit_rate", + "-show_streams", + "-of", + "json", + path.to_str().ok_or_else(|| "invalid_path".to_string())?, + ], + deadline, + ) + .await?; + let warnings_output = run_command_with_deadline( + "ffmpeg", + &[ + "-hide_banner", + "-nostdin", + "-v", + "warning", + "-i", + path.to_str().ok_or_else(|| "invalid_path".to_string())?, + "-map", + "0", + "-c", + "copy", + "-f", + "null", + "-", + ], + deadline, + ) + .await + .unwrap_or_else(|stderr| stderr); + parse_probe(&probe_output, parse_ffmpeg_warnings(&warnings_output)) +} + +async fn run_remux( + input_path: &Path, + output_path: &Path, + mode: HlsSegmentRepairMode, + stream_selection: &RepairRemuxStreamSelection, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; + let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; + let mut args = ["-hide_banner", "-nostdin", "-y", "-copyts", "-i", input] + .into_iter() + .map(ToOwned::to_owned) + .collect::>(); + for stream_index in &stream_selection.mapped_streams { + args.push("-map".to_string()); + args.push(format!("0:{stream_index}")); + } + args.extend(["-c", "copy"].into_iter().map(ToOwned::to_owned)); + if matches!(mode, HlsSegmentRepairMode::Medium | HlsSegmentRepairMode::High) { + args.push("-bsf:v".to_string()); + args.push("dump_extra=freq=keyframe".to_string()); + } + args.push("-mpegts_flags".to_string()); + args.push( + if mode == HlsSegmentRepairMode::High { "+resend_headers+pat_pmt_at_frames" } else { "+resend_headers" } + .to_string(), + ); + args.extend( + ["-mpegts_copyts", "1", "-muxpreload", "0", "-muxdelay", "0", "-f", "mpegts", output] + .into_iter() + .map(ToOwned::to_owned), + ); + let args = args.iter().map(String::as_str).collect::>(); + run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) +} + +pub(super) async fn run_command_with_deadline( + binary: &str, + args: &[&str], + deadline: &HlsPostProcessingDeadline, +) -> Result { + let Some(remaining) = deadline.remaining() else { + return Err("timeout".to_string()); + }; + let output = timeout(remaining, { + let mut command = Command::new(binary); + command.args(args).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()).kill_on_drop(true); + command.output() + }) + .await + .map_err(|_| "timeout".to_string())? + .map_err( + |err| { + if err.kind() == io::ErrorKind::NotFound { + "unsupported".to_string() + } else { + err.to_string() + } + }, + )?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + if output.status.success() { + Ok(if stdout.is_empty() { stderr } else { stdout }) + } else { + Err(if stderr.is_empty() { "command_failed".to_string() } else { stderr }) + } +} + +fn parse_probe(json: &str, warnings: WarningCounters) -> Result { + let value = serde_json::from_str::(json).map_err(|_| "invalid_probe_json".to_string())?; + let streams = value.get("streams").and_then(Value::as_array).ok_or_else(|| "missing_streams".to_string())?; + let mut probe = SegmentProbe { stream_count: streams.len(), warnings, ..SegmentProbe::default() }; + if let Some(format) = value.get("format") { + probe.duration_ms = format.get("duration").and_then(Value::as_str).and_then(parse_seconds_ms_u64); + probe.size = + format.get("size").and_then(Value::as_str).and_then(|value| value.parse().ok()).unwrap_or_default(); + } + for stream in streams { + let codec_type = stream.get("codec_type").and_then(Value::as_str); + let codec_name = stream.get("codec_name").and_then(Value::as_str).map(ToOwned::to_owned); + let start_time_ms = stream.get("start_time").and_then(Value::as_str).and_then(parse_seconds_ms_i64); + let extradata_size = stream + .get("extradata_size") + .and_then(Value::as_u64) + .or_else(|| stream.get("extradata_size").and_then(Value::as_str).and_then(|value| value.parse().ok())); + let stream_index = stream + .get("index") + .and_then(parse_u32_value) + .map_or(probe.streams.len(), |value| value as usize); + let probe_stream = SegmentProbeStream { + index: stream_index, + stream_type: match codec_type { + Some("video") => SegmentProbeStreamType::Video, + Some("audio") => SegmentProbeStreamType::Audio, + _ => SegmentProbeStreamType::Other, + }, + codec_name: codec_name.clone(), + width: stream.get("width").and_then(parse_u32_value), + height: stream.get("height").and_then(parse_u32_value), + sample_rate: stream.get("sample_rate").and_then(parse_u32_value), + channels: stream.get("channels").and_then(parse_u32_value), + }; + match codec_type { + Some("video") if probe.primary_video_codec.is_none() => { + probe.primary_video_codec = codec_name; + probe.primary_video_start_time_ms = start_time_ms; + probe.primary_video_extradata_size = extradata_size; + } + Some("audio") if probe.primary_audio_codec.is_none() => { + probe.primary_audio_codec = codec_name; + probe.primary_audio_start_time_ms = start_time_ms; + } + _ => {} + } + probe.streams.push(probe_stream); + } + Ok(probe) +} + +fn parse_u32_value(value: &Value) -> Option { + value.as_u64().and_then(|value| u32::try_from(value).ok()).or_else(|| { + value + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty() && *value != "N/A") + .and_then(|value| value.parse::().ok()) + }) +} + +fn detect_video_codec(probe: &SegmentProbe) -> RepairVideoCodec { + match probe.primary_video_codec.as_deref() { + Some("h264") => RepairVideoCodec::H264, + Some("hevc" | "h265") => RepairVideoCodec::Hevc, + _ => RepairVideoCodec::Unsupported, + } +} + +fn select_repair_remux_streams(probe: &SegmentProbe) -> Result { + let mut mapped_streams = Vec::new(); + let mut dropped_streams = Vec::new(); + let mut has_video = false; + for stream in &probe.streams { + match stream.stream_type { + SegmentProbeStreamType::Video if valid_video_stream(stream) => { + has_video = true; + mapped_streams.push(stream.index); + } + SegmentProbeStreamType::Audio if valid_audio_stream(stream) => { + mapped_streams.push(stream.index); + } + SegmentProbeStreamType::Video => dropped_streams.push(RepairRemuxDroppedStream { + index: stream.index, + reason: "invalid-video-parameters", + }), + SegmentProbeStreamType::Audio => dropped_streams.push(RepairRemuxDroppedStream { + index: stream.index, + reason: "invalid-audio-parameters", + }), + SegmentProbeStreamType::Other => dropped_streams.push(RepairRemuxDroppedStream { + index: stream.index, + reason: "unsupported-stream-type", + }), + } + } + if !has_video { + return Err("no_valid_video_stream".to_string()); + } + Ok(RepairRemuxStreamSelection { mapped_streams, dropped_streams }) +} + +fn valid_video_stream(stream: &SegmentProbeStream) -> bool { + stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) + && stream.width.unwrap_or_default() > 0 + && stream.height.unwrap_or_default() > 0 +} + +fn valid_audio_stream(stream: &SegmentProbeStream) -> bool { + stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) + && stream.sample_rate.unwrap_or_default() > 0 + && stream.channels.unwrap_or_default() > 0 +} + +fn validate_repair( + config: &HlsSegmentRepairConfig, + codec: RepairVideoCodec, + raw: &SegmentProbe, + fixed: &SegmentProbe, + executed_level: HlsSegmentRepairMode, + stream_selection: &RepairRemuxStreamSelection, +) -> Result<(), String> { + if codec == RepairVideoCodec::Unsupported { + return Err("unsupported_codec".to_string()); + } + let expected_stream_count = if stream_selection.dropped_streams.is_empty() { + raw.stream_count + } else { + stream_selection.mapped_streams.len() + }; + if expected_stream_count != fixed.stream_count { + return Err("stream_count_changed".to_string()); + } + if raw.primary_video_codec != fixed.primary_video_codec { + return Err("video_codec_changed".to_string()); + } + if raw.primary_audio_codec != fixed.primary_audio_codec { + return Err("audio_codec_changed".to_string()); + } + if delta_u64(raw.duration_ms, fixed.duration_ms) > 250 { + return Err("duration_delta_too_large".to_string()); + } + if delta_i64(raw.primary_video_start_time_ms, fixed.primary_video_start_time_ms) > 250 { + return Err("video_start_time_delta_too_large".to_string()); + } + if delta_i64(raw.primary_audio_start_time_ms, fixed.primary_audio_start_time_ms) > 250 { + return Err("audio_start_time_delta_too_large".to_string()); + } + let raw_decision = decide_repair(codec, &raw.warnings); + let fixed_decision = decide_repair(codec, &fixed.warnings); + if raw_decision.required_level != HlsSegmentRepairMode::Off + && fixed_decision.required_level != HlsSegmentRepairMode::Off + { + return Err("repair_triggers_remaining".to_string()); + } + if raw.size > 0 { + let allowed = + raw.size.saturating_add(raw.size.saturating_mul(size_increase_percent(config, executed_level)) / 100); + if fixed.size > allowed { + return Err("size_increase_too_large".to_string()); + } + } + Ok(()) +} + +fn size_increase_percent(config: &HlsSegmentRepairConfig, level: HlsSegmentRepairMode) -> u64 { + match level { + HlsSegmentRepairMode::Off => 0, + HlsSegmentRepairMode::Low => u64::from(config.size_increase.low_percent), + HlsSegmentRepairMode::Medium => u64::from(config.size_increase.medium_percent), + HlsSegmentRepairMode::High => u64::from(config.size_increase.high_percent), + } +} + +fn parse_seconds_ms_u64(value: &str) -> Option { + let (whole, frac) = value.split_once('.').unwrap_or((value, "")); + let whole_ms = whole.parse::().ok()?.checked_mul(1_000)?; + let frac_ms = frac.chars().take(3).collect::(); + let frac_ms = format!("{frac_ms:0<3}").parse::().ok()?; + Some(whole_ms.saturating_add(frac_ms)) +} + +fn parse_seconds_ms_i64(value: &str) -> Option { + let negative = value.starts_with('-'); + let unsigned = value.strip_prefix('-').unwrap_or(value); + let parsed = i64::try_from(parse_seconds_ms_u64(unsigned)?).ok()?; + Some(if negative { -parsed } else { parsed }) +} + +fn delta_u64(lhs: Option, rhs: Option) -> u64 { + match (lhs, rhs) { + (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), + (None, None) => 0, + _ => u64::MAX, + } +} + +fn delta_i64(lhs: Option, rhs: Option) -> u64 { + match (lhs, rhs) { + (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), + (None, None) => 0, + _ => u64::MAX, + } +} + +pub(super) async fn sha256_file(path: &Path) -> io::Result { + let mut file = File::open(path).await?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +pub(super) fn ffmpeg_identity_version() -> String { "system".to_string() } + +fn repair_output_path(raw_path: &Path) -> PathBuf { + let suffix = fastrand::u64(..); + let file_name = raw_path.file_name().and_then(|file_name| file_name.to_str()).unwrap_or("segment"); + raw_path.with_file_name(format!("{file_name}.repair.tmp.{suffix:016x}")) +} + +impl fmt::Display for RepairStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Clean => f.write_str("clean"), + Self::Fixed => f.write_str("fixed"), + Self::PolicyLimited => f.write_str("policy_limited"), + Self::Unsupported => f.write_str("unsupported"), + Self::Timeout => f.write_str("timeout"), + Self::RemuxFailed => f.write_str("remux_failed"), + Self::ValidationFailed => f.write_str("validation_failed"), + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + detect_video_codec, parse_ffmpeg_warnings, parse_probe, repair_object_metadata_key, select_repair_remux_streams, + sha256_file, validate_repair, HlsRepairObjectMetadata, HlsRepairRenderedObjectId, HlsSegmentRepairManager, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, RepairIdentity, RepairRemuxStreamSelection, + RepairStatus, RepairVideoCodec, WarningCounters, REPAIR_METADATA_MAX_ENTRIES, + }; + use crate::{ + api::model::{ + HlsAccessLeaseId, HlsSegmentCache, ProxySessionId, SegmentCacheKey, TransientObjectCacheKey, + TransientResourceId, + }, + model::HlsSegmentRepairConfig, + }; + use std::sync::Arc; + use shared::model::HlsSegmentRepairMode; + + fn repair_config(mode: HlsSegmentRepairMode, apply_to_first_segments: u8) -> HlsSegmentRepairConfig { + HlsSegmentRepairConfig { + max_level: mode, + apply_to_first_segments, + max_parallel_repairs: 1, + ..Default::default() + } + } + + fn should_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> bool { + super::decide_repair(codec, warnings).required_level != HlsSegmentRepairMode::Off + } + + fn repair_context(lease_id: &str, resource_id: &str) -> HlsSegmentRepairObjectContext { + HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + proxy_session_id: ProxySessionId("proxy-session".to_string()), + hls_access_lease_id: Some(HlsAccessLeaseId(lease_id.to_string())), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: resource_id.parse().unwrap_or(1) }, + resource_id: resource_id.to_string(), + file_ext: "ts".to_string(), + origin_fetch_uri_for_diagnostics: format!("http://origin.example/{resource_id}.ts"), + media_sequence: Some(1), + discontinuity_sequence: Some(0), + complete_object: true, + encrypted: false, + custom_response: false, + } + } + + async fn selected_repair_mode( + manager: &HlsSegmentRepairManager, + context: &HlsSegmentRepairObjectContext, + ) -> Option { + manager.try_select_candidate(context).await.map(|(mode, _)| mode) + } + + #[tokio::test] + async fn update_config_applies_to_new_access_lease_windows() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 2)); + let mut updated = repair_config(HlsSegmentRepairMode::Medium, 3); + updated.max_parallel_repairs = 2; + manager.update_config(updated); + + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + + for resource_id in ["1", "2", "3"] { + assert!( + manager.try_select_candidate(&repair_context("lease-a", resource_id)).await.is_some(), + "resource {resource_id} should be inside updated repair window" + ); + } + assert!( + manager.try_select_candidate(&repair_context("lease-a", "4")).await.is_none(), + "fourth resource should be outside updated repair window" + ); + } + + #[test] + fn warning_parser_expands_repeated_messages() { + let warnings = parse_ffmpeg_warnings( + "non-existing SPS 0 referenced in buffering period\nLast message repeated 2 times\nno frame!\n", + ); + + assert_eq!(warnings.missing_sps, 3); + assert_eq!(warnings.no_frame, 1); + } + + #[test] + fn mmco_warning_alone_does_not_trigger_repair() { + let warnings = WarningCounters { mmco_unref_short_failure: 20, ..WarningCounters::default() }; + + assert!(!should_repair(RepairVideoCodec::H264, &warnings)); + } + + #[test] + fn critical_warning_triggers_repair() { + let warnings = WarningCounters { missing_sps: 1, ..WarningCounters::default() }; + + assert!(should_repair(RepairVideoCodec::H264, &warnings)); + } + + #[test] + fn hevc_pps_warning_triggers_repair() { + let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] PPS id out of range: 0\n"); + + assert_eq!(warnings.pps_id_out_of_range, 1); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn hevc_invalid_slice_nalus_with_parameter_issue_trigger_repair() { + let warnings = parse_ffmpeg_warnings( + "[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 0\n[hevc @ 0x1] Skipping invalid undecodable NALU: 1\n", + ); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 2); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 2); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn hevc_keyframe_nalu_counts_as_vcl_but_does_not_trigger_alone() { + let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 21\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_keyframe, 1); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn hevc_metadata_and_dolby_warnings_do_not_trigger_repair_alone() { + let warnings = parse_ffmpeg_warnings( + "[hevc @ 0x1] Skipping invalid undecodable NALU: 39\nMultiple Dolby Vision RPUs found in one AU. Skipping previous.\nAudio/Video desynchronisation detected!\n", + ); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_metadata, 1); + assert_eq!(warnings.dolby_vision_rpu, 1); + assert_eq!(warnings.av_desync, 1); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn hevc_repeated_messages_expand_counters_without_triggering_alone() { + let warnings = + parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 0\nLast message repeated 2 times\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 3); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 3); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn hevc_invalid_undecodable_nalu_0_to_31_counts_as_vcl_trigger_input() { + let warnings = + parse_ffmpeg_warnings("[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 30\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 1); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); + } + + #[test] + fn warning_parser_matches_trigger_patterns_case_insensitively() { + let warnings = parse_ffmpeg_warnings( + "NON-EXISTING SPS 0 referenced\ninvalid nal unit 1\ncould not find codec parameters for stream 0\n", + ); + + assert_eq!(warnings.missing_sps, 1); + assert_eq!(warnings.invalid_nal, 1); + assert_eq!(warnings.codec_parameters_missing, 1); + assert!(should_repair(RepairVideoCodec::H264, &warnings)); + } + + #[test] + fn unsupported_codec_never_triggers_repair() { + let warnings = WarningCounters { missing_sps: 1, pps_id_out_of_range: 1, ..WarningCounters::default() }; + + assert!(!should_repair(RepairVideoCodec::Unsupported, &warnings)); + } + + #[test] + fn parse_probe_detects_hevc_codec_and_extradata() { + let probe = parse_probe( + r#"{ + "streams": [ + { + "index": 0, + "codec_type": "video", + "codec_name": "hevc", + "start_time": "1.250000", + "extradata_size": 96 + } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + assert_eq!(detect_video_codec(&probe), RepairVideoCodec::Hevc); + assert_eq!(probe.primary_video_extradata_size, Some(96)); + } + + #[test] + fn repair_remux_selection_drops_invalid_audio_side_stream() { + let probe = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080 }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 }, + { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0 } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + let selection = select_repair_remux_streams(&probe).expect("valid video should allow remux"); + + assert_eq!(selection.mapped_streams, vec![0, 1]); + assert_eq!(selection.dropped_streams.len(), 1); + assert_eq!(selection.dropped_streams[0].index, 2); + assert_eq!(selection.dropped_streams[0].reason, "invalid-audio-parameters"); + } + + #[test] + fn repair_validation_allows_configured_stream_drop() { + let raw = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" }, + { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0, "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { codec_parameters_missing: 1, ..WarningCounters::default() }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("fixed probe should parse"); + let selection = select_repair_remux_streams(&raw).expect("raw should select valid streams"); + + assert!(validate_repair( + &repair_config(HlsSegmentRepairMode::Low, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::Low, + &selection + ) + .is_ok()); + } + + #[test] + fn repair_remux_selection_rejects_without_valid_video() { + let probe = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "height": 1080 }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + let err = select_repair_remux_streams(&probe).expect_err("missing width should reject video"); + + assert_eq!(err, "no_valid_video_stream"); + } + + #[test] + fn hevc_validation_accepts_when_repair_triggers_are_removed() { + let raw = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { + pps_id_out_of_range: 1, + invalid_undecodable_nalu_non_metadata: 2, + invalid_undecodable_nalu_metadata: 1, + dolby_vision_rpu: 1, + ..WarningCounters::default() + }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1010" } + }"#, + WarningCounters { invalid_undecodable_nalu_metadata: 1, dolby_vision_rpu: 1, ..WarningCounters::default() }, + ) + .expect("fixed probe should parse"); + + assert!(validate_repair( + &repair_config(HlsSegmentRepairMode::Medium, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::Medium, + &RepairRemuxStreamSelection::preserve_all(&raw) + ) + .is_ok()); + } + + #[test] + fn validation_rejects_remaining_repair_triggers_even_when_level_improves() { + let raw = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { codec_parameters_missing: 1, missing_sps: 1, ..WarningCounters::default() }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { missing_sps: 1, ..WarningCounters::default() }, + ) + .expect("fixed probe should parse"); + + let err = validate_repair( + &repair_config(HlsSegmentRepairMode::High, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::High, + &RepairRemuxStreamSelection::preserve_all(&raw), + ) + .expect_err("remaining medium trigger should fail validation"); + assert_eq!(err, "repair_triggers_remaining"); + } + + #[test] + fn configured_max_below_required_skips_repair() { + assert_eq!( + HlsSegmentRepairMode::Low.execution_plan(HlsSegmentRepairMode::High), + super::HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired + ); + assert_eq!( + HlsSegmentRepairMode::Medium.execution_plan(HlsSegmentRepairMode::Low), + super::HlsSegmentRepairExecutionPlan::Repair(HlsSegmentRepairMode::Low) + ); + assert_eq!( + HlsSegmentRepairMode::Off.execution_plan(HlsSegmentRepairMode::High), + super::HlsSegmentRepairExecutionPlan::SkipNoTrigger + ); + } + + #[tokio::test] + async fn repair_window_selects_first_unique_segments_per_access_lease() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let first = repair_context("lease-a", "000001"); + let second = repair_context("lease-a", "000002"); + + assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &first).await, None); + assert_eq!(selected_repair_mode(&manager, &second).await, None); + } + + #[tokio::test] + async fn background_candidate_without_access_lease_is_ignored_before_window_check() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut context = repair_context("lease-a", "000001"); + context.hls_access_lease_id = None; + + assert_eq!(selected_repair_mode(&manager, &context).await, None); + + let stats = manager.stats().await; + assert_eq!(stats.windows, 1); + assert_eq!(stats.checked_candidates, 0); + assert_eq!(stats.object_metadata, 0); + } + + #[tokio::test] + async fn repair_window_is_separate_per_access_lease() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, + Some(HlsSegmentRepairMode::Medium) + ); + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-b", "000001")).await, + Some(HlsSegmentRepairMode::Medium) + ); + } + + #[tokio::test] + async fn known_normal_object_consumes_new_lease_window_without_rescan() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let cache_key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 1, "ts"); + let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-normal-bytes").await.expect("commit"); + let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); + let previous_context = repair_context("lease-a", "1"); + manager + .record_object_metadata( + repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256: Some("previous-raw".to_string()), + status: RepairStatus::Clean, + raw_size: metadata.size, + final_size: metadata.size, + validation_reason: None, + }, + ) + .await; + + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + + assert!(manager + .repair_ready_cache_hit(&cache, &cache_key, repair_context("lease-b", "1")) + .await + .expect("repair cache hit") + .is_none()); + assert_eq!(manager.stats().await.metadata, 0); + assert_eq!(selected_repair_mode(&manager, &repair_context("lease-b", "2")).await, None); + } + + #[tokio::test] + async fn known_transient_object_consumes_new_lease_window_without_rescan() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let cache_key = TransientObjectCacheKey::new( + ProxySessionId("proxy-session".to_string()), + TransientResourceId("resource-a".to_string()), + "ts", + ); + let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-transient-bytes").await.expect("commit"); + let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); + let mut previous_context = repair_context("lease-a", "1"); + previous_context.source = HlsSegmentRepairSource::Transient; + previous_context.rendered_object_id = + HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; + previous_context.resource_id = "resource-a".to_string(); + manager + .record_object_metadata( + repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256: Some("previous-raw".to_string()), + status: RepairStatus::Clean, + raw_size: metadata.size, + final_size: metadata.size, + validation_reason: None, + }, + ) + .await; + + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + let mut current_context = previous_context.clone(); + current_context.hls_access_lease_id = Some(HlsAccessLeaseId("lease-b".to_string())); + + assert!(manager + .repair_ready_cache_hit(&cache, &cache_key, current_context) + .await + .expect("repair cache hit") + .is_none()); + assert_eq!(manager.stats().await.metadata, 0); + let mut second_context = repair_context("lease-b", "2"); + second_context.source = HlsSegmentRepairSource::Transient; + second_context.rendered_object_id = + HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; + second_context.resource_id = "resource-b".to_string(); + assert_eq!(selected_repair_mode(&manager, &second_context).await, None); + } + + #[tokio::test] + async fn object_metadata_hash_mismatch_does_not_skip_repair_evaluation() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let context = repair_context("lease-a", "1"); + let object_key = repair_object_metadata_key(&context, HlsSegmentRepairMode::Low); + manager + .record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: "old-hash".to_string(), + raw_sha256: Some("old-raw".to_string()), + status: RepairStatus::Clean, + raw_size: 1, + final_size: 1, + validation_reason: None, + }, + ) + .await; + + assert!(!manager.object_metadata_matches(&object_key, "new-hash").await); + } + + #[test] + fn repair_object_metadata_key_ignores_origin_fetch_uri_for_diagnostics() { + let mut first = repair_context("lease-a", "1"); + first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); + let mut second = first.clone(); + second.origin_fetch_uri_for_diagnostics = "http://redirect-b.example/cdn/path/1.ts".to_string(); + + assert_eq!( + repair_object_metadata_key(&first, HlsSegmentRepairMode::Low), + repair_object_metadata_key(&second, HlsSegmentRepairMode::Low) + ); + } + + #[tokio::test] + async fn repair_window_candidate_key_ignores_origin_fetch_uri_for_diagnostics() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut first = repair_context("lease-a", "1"); + first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); + let mut same_rendered_object_other_fetch_uri = first.clone(); + same_rendered_object_other_fetch_uri.origin_fetch_uri_for_diagnostics = + "http://redirect-b.example/cdn/path/1.ts".to_string(); + let second_rendered_object = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &same_rendered_object_other_fetch_uri).await, None); + assert_eq!( + selected_repair_mode(&manager, &second_rendered_object).await, + Some(HlsSegmentRepairMode::Low) + ); + } + + #[tokio::test] + async fn repair_disabled_does_not_track_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Off, 1)); + let context = repair_context("lease-a", "000001"); + + assert_eq!(selected_repair_mode(&manager, &context).await, None); + + let registry = manager.windows.read().await; + assert_eq!(registry.checked_candidates.len(), 0); + } + + #[tokio::test] + async fn transient_commit_and_cache_hit_consume_repair_window_once() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut commit_context = repair_context("lease-a", "1"); + commit_context.source = HlsSegmentRepairSource::Transient; + commit_context.rendered_object_id = + HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; + commit_context.resource_id = "resource-a".to_string(); + commit_context.origin_fetch_uri_for_diagnostics = "http://origin.example/live/resource-a.ts".to_string(); + commit_context.media_sequence = None; + commit_context.discontinuity_sequence = None; + + let mut cache_hit_context = commit_context.clone(); + cache_hit_context.origin_fetch_uri_for_diagnostics = "resource-a".to_string(); + + let mut second_context = commit_context.clone(); + second_context.rendered_object_id = + HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; + second_context.resource_id = "resource-b".to_string(); + second_context.origin_fetch_uri_for_diagnostics = "resource-b".to_string(); + + assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); + assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); + } + + #[tokio::test] + async fn normal_commit_and_cache_hit_consume_repair_window_once() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let commit_context = repair_context("lease-a", "1"); + let mut cache_hit_context = commit_context.clone(); + cache_hit_context.origin_fetch_uri_for_diagnostics = "http://redirect.example/other-path.ts".to_string(); + cache_hit_context.media_sequence = Some(99); + cache_hit_context.discontinuity_sequence = Some(7); + let second_context = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); + assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); + } + + #[tokio::test] + async fn non_repairable_objects_do_not_consume_repair_window() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut partial = repair_context("lease-a", "1"); + partial.complete_object = false; + let repairable = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &partial).await, None); + assert_eq!(selected_repair_mode(&manager, &repairable).await, Some(HlsSegmentRepairMode::Low)); + } + + #[tokio::test] + async fn new_repair_window_generation_allows_rechecking_candidate() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let context = repair_context("lease-a", "000001"); + + manager.start_access_lease_window(lease_id.clone()).await; + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.start_access_lease_window(lease_id).await; + + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 2); + } + + #[tokio::test] + async fn remove_access_lease_window_clears_window_and_generation() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.start_access_lease_window(lease_id.clone()).await; + + let before = manager.stats().await; + assert_eq!(before.windows, 1); + assert_eq!(before.generations, 1); + + manager.remove_access_lease_window(&lease_id).await; + + let after = manager.stats().await; + assert_eq!(after.windows, 0); + assert_eq!(after.generations, 0); + } + + #[tokio::test] + async fn remove_access_lease_window_clears_checked_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + + manager.start_access_lease_window(lease_id.clone()).await; + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, + Some(HlsSegmentRepairMode::Low) + ); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.remove_access_lease_window(&lease_id).await; + + assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); + } + + #[tokio::test] + async fn remove_proxy_session_state_clears_checked_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.start_access_lease_window(lease_id.clone()).await; + let context = repair_context("lease-a", "000001"); + + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; + + assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); + } + + #[tokio::test] + async fn remove_proxy_session_state_clears_object_metadata() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let context = repair_context("lease-a", "000001"); + manager + .record_object_metadata( + repair_object_metadata_key(&context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256: "hash".to_string(), + raw_sha256: Some("raw".to_string()), + status: RepairStatus::Clean, + raw_size: 1, + final_size: 1, + validation_reason: None, + }, + ) + .await; + + assert_eq!(manager.stats().await.object_metadata, 1); + + manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; + + assert_eq!(manager.stats().await.object_metadata, 0); + } + + #[tokio::test] + async fn repair_metadata_is_bounded() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + + for index in 0..REPAIR_METADATA_MAX_ENTRIES + 5 { + manager + .record_metadata( + RepairIdentity { + raw_sha256: format!("{index:064x}"), + repair_mode: HlsSegmentRepairMode::Low, + command_version: 1, + ffmpeg_version: "test".to_string(), + }, + RepairStatus::Clean, + 1, + 1, + None, + ) + .await; + } + + assert_eq!(manager.stats().await.metadata, REPAIR_METADATA_MAX_ENTRIES); + } + + #[tokio::test] + async fn repair_lock_cleanup_keeps_waited_lock_and_removes_unused_lock() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let identity = RepairIdentity { + raw_sha256: "a".repeat(64), + repair_mode: HlsSegmentRepairMode::Low, + command_version: 1, + ffmpeg_version: "test".to_string(), + }; + let lock = manager.lock_for_identity(identity.clone()).await; + let waiter = Arc::clone(&lock); + + manager.remove_lock_if_unused(&identity, &lock).await; + assert_eq!(manager.stats().await.locks, 1); + + drop(waiter); + manager.remove_lock_if_unused(&identity, &lock).await; + assert_eq!(manager.stats().await.locks, 0); + } + + #[test] + fn repair_context_excludes_non_finite_origin_ts_objects() { + let mut context = repair_context("lease-a", "000001"); + + context.complete_object = false; + assert!(!context.is_repairable_ts()); + + context.complete_object = true; + context.encrypted = true; + assert!(!context.is_repairable_ts()); + + context.encrypted = false; + context.custom_response = true; + assert!(!context.is_repairable_ts()); + + context.custom_response = false; + context.file_ext = "m4s".to_string(); + assert!(!context.is_repairable_ts()); + } +} diff --git a/backend/src/api/model/hls_cache/segment_watchdog.rs b/backend/src/api/model/hls_cache/segment_watchdog.rs new file mode 100644 index 000000000..32216cc5f --- /dev/null +++ b/backend/src/api/model/hls_cache/segment_watchdog.rs @@ -0,0 +1,736 @@ +#![allow(clippy::large_futures)] + +use super::{ + segment_repair::{ + ffmpeg_identity_version, run_command_with_deadline, sha256_file, HlsPostProcessingDeadline, + HlsSegmentRepairObjectContext, + }, + CachedSegmentMetadata, HlsCacheObjectKey, HlsSegmentCache, StagedCacheObject, +}; +use crate::model::HlsCorruptSegmentWatchdogConfig; +use shared::model::HlsCorruptSegmentWatchdogMode; +use log::debug; +use serde_json::Value; +use std::{ + collections::{HashMap, VecDeque}, + ffi::OsString, + io, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::{ + fs, + sync::{Mutex, RwLock, Semaphore}, +}; + +const WATCHDOG_COMMAND_VERSION: u32 = 1; +const WATCHDOG_METADATA_MAX_ENTRIES: usize = 4_096; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum HlsCorruptSegmentWatchdogStatus { + Clean, + DetectedCorrupt { packet_corrupt_count: u32 }, + Sanitized { packet_corrupt_before: u32, packet_corrupt_after: u32 }, + DiagnosticSanitized { packet_corrupt_before: u32, packet_corrupt_after: u32 }, + UnsupportedContainer { extension: String }, + Timeout, + SanitizeFailed { reason: String }, + ValidationFailed { reason: String }, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +struct HlsWatchdogSanitizeArtifactKey { + raw_sha256: String, + command_version: u32, + ffmpeg_version: String, +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct HlsWatchdogArtifactMetadata { + status: HlsCorruptSegmentWatchdogStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct HlsCorruptSegmentWatchdogStats { + pub metadata: usize, + pub locks: usize, +} + +#[derive(Debug, Default)] +pub struct HlsCorruptSegmentWatchdogManager { + metadata: RwLock>, + metadata_order: Mutex>, + locks: Mutex>>>, +} + +impl HlsCorruptSegmentWatchdogManager { + pub fn new() -> Self { Self::default() } + + pub async fn clear_runtime_state(&self) { + self.metadata.write().await.clear(); + self.metadata_order.lock().await.clear(); + self.locks.lock().await.clear(); + } + + pub async fn stats(&self) -> HlsCorruptSegmentWatchdogStats { + HlsCorruptSegmentWatchdogStats { + metadata: self.metadata.read().await.len(), + locks: self.locks.lock().await.len(), + } + } + + #[allow(clippy::too_many_arguments)] + pub async fn process_staged_and_commit( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + context: &HlsSegmentRepairObjectContext, + config: &HlsCorruptSegmentWatchdogConfig, + semaphore: &Arc, + raw_sha256: String, + deadline: &HlsPostProcessingDeadline, + ) -> io::Result + where + K: HlsCacheObjectKey, + { + if !config.mode.is_enabled() { + return segment_cache.commit_staged(key, raw).await; + } + let identity = HlsWatchdogSanitizeArtifactKey { + raw_sha256: raw_sha256.clone(), + command_version: WATCHDOG_COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + }; + if let Some(metadata) = self.metadata(&identity).await { + if !matches!( + metadata.status, + HlsCorruptSegmentWatchdogStatus::Sanitized { .. } + | HlsCorruptSegmentWatchdogStatus::DiagnosticSanitized { .. } + ) { + debug_watchdog_event(context, config.mode, "metadata hit", Some(status_log_value(&metadata.status))); + return segment_cache.commit_staged(key, raw).await; + } + } + let Some(remaining) = deadline.remaining() else { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some("timeout".to_string()), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + }; + let permit = match tokio::time::timeout(remaining, semaphore.acquire()).await { + Ok(Ok(permit)) => permit, + Ok(Err(_)) => return segment_cache.commit_staged(key, raw).await, + Err(_) => { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some("timeout".to_string()), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + }; + let lock = self.lock_for_identity(identity.clone()).await; + let result = { + let _permit = permit; + if let Some(remaining) = deadline.remaining() { + if let Ok(_guard) = tokio::time::timeout(remaining, lock.lock()).await { + self.process_locked( + segment_cache, + key, + raw, + context, + config, + identity.clone(), + raw_sha256, + deadline, + ) + .await + } else { + self.record_metadata( + identity.clone(), + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some("timeout".to_string()), + ) + .await; + segment_cache.commit_staged(key, raw).await + } + } else { + self.record_metadata( + identity.clone(), + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some("timeout".to_string()), + ) + .await; + segment_cache.commit_staged(key, raw).await + } + }; + self.remove_lock_if_unused(&identity, &lock).await; + result + } + + #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_lines)] + async fn process_locked( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + context: &HlsSegmentRepairObjectContext, + config: &HlsCorruptSegmentWatchdogConfig, + identity: HlsWatchdogSanitizeArtifactKey, + raw_sha256: String, + deadline: &HlsPostProcessingDeadline, + ) -> io::Result + where + K: HlsCacheObjectKey, + { + if let Some(metadata) = self.metadata(&identity).await { + if !matches!( + metadata.status, + HlsCorruptSegmentWatchdogStatus::Sanitized { .. } + | HlsCorruptSegmentWatchdogStatus::DiagnosticSanitized { .. } + ) { + return segment_cache.commit_staged(key, raw).await; + } + } + let raw_detect = match detect_packet_corrupt(&raw.path, deadline).await { + Ok(count) => count, + Err(reason) if reason == "timeout" => { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + Err(reason) => { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + }; + if raw_detect == 0 { + self.record_metadata(identity, HlsCorruptSegmentWatchdogStatus::Clean, raw.size, raw.size, None).await; + debug_watchdog_event(context, config.mode, "clean", Some("packet_corrupt=0")); + return segment_cache.commit_staged(key, raw).await; + } + if config.mode == HlsCorruptSegmentWatchdogMode::DetectOnly { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::DetectedCorrupt { packet_corrupt_count: raw_detect }, + raw.size, + raw.size, + None, + ) + .await; + debug_watchdog_event(context, config.mode, "detected corrupt", Some("action=raw_commit")); + return segment_cache.commit_staged(key, raw).await; + } + if let Some(extension) = unsupported_sanitize_extension(&key.file_name()) { + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::UnsupportedContainer { extension: extension.clone() }, + raw.size, + raw.size, + Some(format!("unsupported_container:{extension}")), + ) + .await; + debug_watchdog_event(context, config.mode, "detected corrupt", Some("action=raw_commit reason=unsupported_container")); + return segment_cache.commit_staged(key, raw).await; + } + let fixed_path = watchdog_output_path(&raw.path); + if let Err(reason) = sanitize_corrupt_segment(&raw.path, &fixed_path, deadline).await { + let _ = fs::remove_file(&fixed_path).await; + let status = if reason == "timeout" { + HlsCorruptSegmentWatchdogStatus::Timeout + } else { + HlsCorruptSegmentWatchdogStatus::SanitizeFailed { reason: reason.clone() } + }; + self.record_metadata(identity, status, raw.size, raw.size, Some(reason)).await; + return segment_cache.commit_staged(key, raw).await; + } + let fixed_size = match fs::metadata(&fixed_path).await { + Ok(metadata) => metadata.len(), + Err(err) => { + let reason = format!("metadata_failed:{err}"); + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + }; + let fixed_detect = match detect_packet_corrupt(&fixed_path, deadline).await { + Ok(count) => count, + Err(reason) if reason == "timeout" => { + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::Timeout, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + Err(reason) => { + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + }; + let validation = validate_sanitized_segment( + &raw.path, + raw.size, + &fixed_path, + fixed_size, + fixed_detect, + config.mode, + deadline, + ) + .await; + if let Err(reason) = validation { + let _ = fs::remove_file(&fixed_path).await; + self.record_metadata( + identity, + HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, + raw.size, + raw.size, + Some(reason), + ) + .await; + return segment_cache.commit_staged(key, raw).await; + } + let _ = segment_cache.remove_staged(raw.clone()).await; + let committed = + segment_cache.commit_staged(key, StagedCacheObject { path: fixed_path, size: fixed_size }).await?; + let status = if config.mode == HlsCorruptSegmentWatchdogMode::Diagnostic { + HlsCorruptSegmentWatchdogStatus::DiagnosticSanitized { + packet_corrupt_before: raw_detect, + packet_corrupt_after: fixed_detect, + } + } else { + HlsCorruptSegmentWatchdogStatus::Sanitized { + packet_corrupt_before: raw_detect, + packet_corrupt_after: fixed_detect, + } + }; + let committed_sha = sha256_file(&committed.path).await.unwrap_or(raw_sha256); + self.record_metadata(identity, status, raw.size, committed.size, Some(committed_sha)).await; + debug_watchdog_event(context, config.mode, "sanitized", Some("action=fixed_commit")); + Ok(committed) + } + + async fn metadata(&self, identity: &HlsWatchdogSanitizeArtifactKey) -> Option { + self.metadata.read().await.get(identity).cloned() + } + + async fn record_metadata( + &self, + identity: HlsWatchdogSanitizeArtifactKey, + status: HlsCorruptSegmentWatchdogStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, + ) { + let inserted_new = { + let mut metadata = self.metadata.write().await; + let inserted_new = !metadata.contains_key(&identity); + metadata.insert( + identity.clone(), + HlsWatchdogArtifactMetadata { status, raw_size, final_size, validation_reason }, + ); + inserted_new + }; + if inserted_new { + self.metadata_order.lock().await.push_back(identity); + } + self.prune_metadata().await; + } + + async fn lock_for_identity(&self, identity: HlsWatchdogSanitizeArtifactKey) -> Arc> { + let mut locks = self.locks.lock().await; + Arc::clone(locks.entry(identity).or_insert_with(|| Arc::new(Mutex::new(())))) + } + + async fn remove_lock_if_unused(&self, identity: &HlsWatchdogSanitizeArtifactKey, lock: &Arc>) { + let mut locks = self.locks.lock().await; + if Arc::strong_count(lock) <= 2 && locks.get(identity).is_some_and(|current| Arc::ptr_eq(current, lock)) { + locks.remove(identity); + } + } + + async fn prune_metadata(&self) { + loop { + let should_prune = self.metadata.read().await.len() > WATCHDOG_METADATA_MAX_ENTRIES; + if !should_prune { + return; + } + let Some(oldest) = self.metadata_order.lock().await.pop_front() else { + return; + }; + self.metadata.write().await.remove(&oldest); + } + } +} + +async fn detect_packet_corrupt(path: &Path, deadline: &HlsPostProcessingDeadline) -> Result { + let path = path.to_str().ok_or_else(|| "invalid_path".to_string())?; + let output = match run_command_with_deadline( + "ffmpeg", + &["-hide_banner", "-nostdin", "-v", "warning", "-i", path, "-map", "0", "-c", "copy", "-f", "null", "-"], + deadline, + ) + .await + { + Ok(output) => output, + Err(reason) if reason == "timeout" => return Err(reason), + Err(stderr) => stderr, + }; + Ok(count_packet_corrupt_events(&output)) +} + +fn count_packet_corrupt_events(stderr: &str) -> u32 { + let mut dts_values = std::collections::HashSet::new(); + let mut count = 0_u32; + let mut last_increment = 0_u32; + for line in stderr.lines() { + let trimmed = line.trim(); + if let Some(repeated) = parse_repeated_count(trimmed) { + count = count.saturating_add(last_increment.saturating_mul(repeated)); + continue; + } + let lower = trimmed.to_ascii_lowercase(); + if !lower.contains("packet corrupt") { + last_increment = 0; + continue; + } + if let Some(dts) = parse_packet_corrupt_dts(trimmed) { + if dts_values.insert(dts) { + count = count.saturating_add(1); + last_increment = 1; + } else { + last_increment = 0; + } + } else { + count = count.saturating_add(1); + last_increment = 1; + } + } + count +} + +fn parse_repeated_count(line: &str) -> Option { + let rest = line.strip_prefix("Last message repeated ")?; + rest.strip_suffix(" times")?.parse().ok() +} + +fn parse_packet_corrupt_dts(line: &str) -> Option { + let (_, rest) = line.split_once("dts = ")?; + rest.split(|ch: char| !ch.is_ascii_digit() && ch != '-').next().filter(|value| !value.is_empty())?.parse().ok() +} + +async fn sanitize_corrupt_segment( + input_path: &Path, + output_path: &Path, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; + let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; + run_command_with_deadline( + "ffmpeg", + &[ + "-hide_banner", + "-nostdin", + "-y", + "-fflags", + "+discardcorrupt", + "-copyts", + "-i", + input, + "-map", + "0", + "-c", + "copy", + "-mpegts_flags", + "+resend_headers", + "-mpegts_copyts", + "1", + "-muxpreload", + "0", + "-muxdelay", + "0", + "-f", + "mpegts", + output, + ], + deadline, + ) + .await + .map(|_| ()) +} + +async fn validate_sanitized_segment( + raw_path: &Path, + raw_size: u64, + fixed_path: &Path, + fixed_size: u64, + fixed_packet_corrupt: u32, + mode: HlsCorruptSegmentWatchdogMode, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + if fixed_packet_corrupt > 0 { + return Err("packet_corrupt_after_sanitize".to_string()); + } + if fixed_size == 0 { + return Err("fixed_empty".to_string()); + } + if size_increase_percent(raw_size, fixed_size) > 2 { + return Err("fixed_size_increase".to_string()); + } + if mode == HlsCorruptSegmentWatchdogMode::Diagnostic { + validate_diagnostic_metadata(raw_path, fixed_path, deadline).await?; + } + Ok(()) +} + +fn size_increase_percent(raw_size: u64, fixed_size: u64) -> u64 { + if fixed_size <= raw_size { + return 0; + } + if raw_size == 0 { + return 100; + } + fixed_size.saturating_sub(raw_size).saturating_mul(100) / raw_size +} + +#[derive(Debug, Clone, Default)] +struct WatchdogProbeMetadata { + duration_ms: Option, + stream_count: usize, + primary_video_codec: Option, + primary_audio_codec: Option, + primary_video_start_time_ms: Option, + primary_audio_start_time_ms: Option, +} + +async fn validate_diagnostic_metadata( + raw_path: &Path, + fixed_path: &Path, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let raw = probe_metadata(raw_path, deadline).await?; + let fixed = probe_metadata(fixed_path, deadline).await?; + if raw.stream_count != fixed.stream_count { + return Err("stream_count_changed".to_string()); + } + if raw.primary_video_codec != fixed.primary_video_codec { + return Err("primary_video_codec_changed".to_string()); + } + if raw.primary_audio_codec != fixed.primary_audio_codec { + return Err("primary_audio_codec_changed".to_string()); + } + if delta_too_large(raw.primary_video_start_time_ms, fixed.primary_video_start_time_ms, 250) + || delta_too_large(raw.primary_audio_start_time_ms, fixed.primary_audio_start_time_ms, 250) + { + return Err("start_time_delta".to_string()); + } + if delta_too_large(raw.duration_ms, fixed.duration_ms, 500) { + return Err("duration_delta".to_string()); + } + Ok(()) +} + +fn delta_too_large(left: Option, right: Option, max_delta_ms: i64) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.saturating_sub(right).abs() > max_delta_ms, + _ => false, + } +} + +async fn probe_metadata(path: &Path, deadline: &HlsPostProcessingDeadline) -> Result { + let path = path.to_str().ok_or_else(|| "invalid_path".to_string())?; + let output = run_command_with_deadline( + "ffprobe", + &[ + "-hide_banner", + "-v", + "error", + "-show_entries", + "format=duration,size,bit_rate", + "-show_entries", + "stream=index,codec_type,codec_name,start_time,duration,id", + "-of", + "json", + path, + ], + deadline, + ) + .await?; + parse_probe_metadata(&output) +} + +fn parse_probe_metadata(output: &str) -> Result { + let value: Value = serde_json::from_str(output).map_err(|err| err.to_string())?; + let streams = value.get("streams").and_then(Value::as_array).ok_or_else(|| "missing_streams".to_string())?; + let mut metadata = WatchdogProbeMetadata { stream_count: streams.len(), ..WatchdogProbeMetadata::default() }; + for stream in streams { + let codec_type = stream.get("codec_type").and_then(Value::as_str); + let codec_name = stream.get("codec_name").and_then(Value::as_str).map(str::to_string); + let start_time_ms = stream.get("start_time").and_then(Value::as_str).and_then(parse_seconds_to_millis); + match codec_type { + Some("video") if metadata.primary_video_codec.is_none() => { + metadata.primary_video_codec = codec_name; + metadata.primary_video_start_time_ms = start_time_ms; + } + Some("audio") if metadata.primary_audio_codec.is_none() => { + metadata.primary_audio_codec = codec_name; + metadata.primary_audio_start_time_ms = start_time_ms; + } + _ => {} + } + } + metadata.duration_ms = value + .get("format") + .and_then(|format| format.get("duration")) + .and_then(Value::as_str) + .and_then(parse_seconds_to_millis); + Ok(metadata) +} + +#[allow(clippy::cast_possible_truncation)] +fn parse_seconds_to_millis(value: &str) -> Option { + let value: f64 = value.parse().ok()?; + Some((value * 1_000.0).round() as i64) +} + +fn watchdog_output_path(input_path: &Path) -> PathBuf { + let mut value: OsString = input_path.as_os_str().to_os_string(); + value.push(".watchdog.ts"); + PathBuf::from(value) +} + +fn status_log_value(status: &HlsCorruptSegmentWatchdogStatus) -> &'static str { + match status { + HlsCorruptSegmentWatchdogStatus::Clean => "clean", + HlsCorruptSegmentWatchdogStatus::DetectedCorrupt { .. } => "detected_corrupt", + HlsCorruptSegmentWatchdogStatus::Sanitized { .. } => "sanitized", + HlsCorruptSegmentWatchdogStatus::DiagnosticSanitized { .. } => "diagnostic_sanitized", + HlsCorruptSegmentWatchdogStatus::UnsupportedContainer { .. } => "unsupported_container", + HlsCorruptSegmentWatchdogStatus::Timeout => "timeout", + HlsCorruptSegmentWatchdogStatus::SanitizeFailed { .. } => "sanitize_failed", + HlsCorruptSegmentWatchdogStatus::ValidationFailed { .. } => "validation_failed", + } +} + +fn unsupported_sanitize_extension(file_name: &str) -> Option { + let extension = file_name.rsplit_once('.')?.1.to_ascii_lowercase(); + if matches!(extension.as_str(), "ts" | "mpegts") { + None + } else { + Some(extension) + } +} + +fn debug_watchdog_event( + context: &HlsSegmentRepairObjectContext, + mode: HlsCorruptSegmentWatchdogMode, + event: &'static str, + detail: Option<&str>, +) { + if let Some(detail) = detail { + debug!( + "HLS corrupt segment watchdog {event}: session={} source={} resource={} mode={} {detail}", + super::safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ); + } else { + debug!( + "HLS corrupt segment watchdog {event}: session={} source={} resource={} mode={}", + super::safe_proxy_session_id(&context.proxy_session_id), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ); + } +} + +#[cfg(test)] +mod tests { + use super::{count_packet_corrupt_events, parse_packet_corrupt_dts, unsupported_sanitize_extension}; + + #[test] + fn packet_corrupt_counter_dedupes_mpegts_and_hls_same_dts() { + let stderr = r" +[mpegts @ 0x1] Packet corrupt (stream = 0, dts = 5850142200). +[hls @ 0x2] Packet corrupt (stream = 0, dts = 5850142200). +[hls @ 0x2] Packet corrupt (stream = 0, dts = 5851004400). +"; + + assert_eq!(count_packet_corrupt_events(stderr), 2); + } + + #[test] + fn packet_corrupt_counter_expands_repeated_messages() { + let stderr = r" +Packet corrupt +Last message repeated 3 times +"; + + assert_eq!(count_packet_corrupt_events(stderr), 4); + } + + #[test] + fn parses_packet_corrupt_dts() { + assert_eq!(parse_packet_corrupt_dts("[mpegts @ 0x1] Packet corrupt (stream = 0, dts = -42)."), Some(-42)); + } + + #[test] + fn watchdog_sanitize_is_limited_to_ts_like_extensions() { + assert_eq!(unsupported_sanitize_extension("000001.ts"), None); + assert_eq!(unsupported_sanitize_extension("000001.mpegts"), None); + assert_eq!(unsupported_sanitize_extension("000001.m4s"), Some("m4s".to_string())); + assert_eq!(unsupported_sanitize_extension("000001.mp4"), Some("mp4".to_string())); + } +} diff --git a/backend/src/api/model/hls_cache/session.rs b/backend/src/api/model/hls_cache/session.rs new file mode 100644 index 000000000..f97f3a7b5 --- /dev/null +++ b/backend/src/api/model/hls_cache/session.rs @@ -0,0 +1,746 @@ +use super::{ + build_proxy_session_id, classify_account_binding_protection, HlsAccountBindingProtection, HlsAccountOverlapTiming, + HlsEffectiveOriginAcquirePolicy, HlsEffectiveOriginAcquirePolicyState, HlsFreshManifestRequiredReason, + HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountRebindState, HlsOriginSource, HlsSessionKey, + HlsBoundAccountAcquireErrorKind, MapCacheStatus, MapEntry, OriginMapKey, OriginRefreshState, OriginSegmentKey, + ProxyMapId, ProxySessionId, RenderPolicy, RenderedManifest, SegmentCacheStatus, SegmentEntry, + SegmentFetchPriority, SegmentPrefetchQueue, TransientPassthroughState, +}; +use axum::http::{HeaderMap, StatusCode}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + sync::Arc, +}; +use tokio::sync::Notify; + +/// Session mode selected for a shared HLS content session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsSessionMode { + NormalCacheTimeline, + TransientPassthrough { reason: TransientPassthroughReason }, +} + +/// Reason why a session cannot use the normal cache timeline. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TransientPassthroughReason { + ExtXKey, + UnsupportedTag { tag: String }, + ParserUnsupportedFeature { feature: String }, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HlsSegmentFailureTracker { + pub consecutive_temporary_failures: u32, + pub last_failure_at_ms: Option, + pub last_failed_object: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsSegmentFailureObject { + Normal { proxy_seq: u64, origin_seq: u64 }, + Transient { resource_id: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsSegmentFailureTransition { + StillRetryable { failures: u32, threshold: u32 }, + BecamePermanentlyFailed { failures: u32, threshold: u32 }, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HlsManifestTemporaryFailureTracker { + pub consecutive_temporary_failures: u32, + pub last_failure_at_ms: Option, + pub last_failure_kind: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestTemporaryFailureKind { + Timeout, + RetryableStatus { status: StatusCode }, + ProviderAcquire { kind: HlsBoundAccountAcquireErrorKind }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsManifestTemporaryFailureTransition { + StillRetryable { failures: u32, threshold: u32 }, + BecameChannelUnavailable { failures: u32, threshold: u32 }, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HlsManifestAcceptanceState { + pub same_host_retry_chain_failures: u32, + pub host_switch_candidate: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsManifestHostSwitchCandidate { + pub host: String, + pub target_url: String, + pub first_seen_at_ms: u64, + pub last_seen_at_ms: u64, + pub seen_count: u32, + pub highwater: Option, + pub quality_score: u16, +} + +#[derive(Debug, Clone, Default)] +pub struct HlsSessionActivity { + pub last_authorized_manifest_at_ms: Option, + pub last_authorized_media_at_ms: Option, + pub active_access_lease_count: usize, + pub active_origin_work_count: usize, + pub origin_work_generation: u64, +} + +/// Shared runtime state for one stable HLS origin entry. +pub struct HlsSession { + pub key: HlsSessionKey, + pub proxy_session_id: ProxySessionId, + pub origin_source: HlsOriginSource, + pub origin_account_binding: Option, + pub origin_account_io_lease: Option, + pub origin_account_rebind: HlsOriginAccountRebindState, + pub effective_origin_acquire_policy: Option, + pub mode: HlsSessionMode, + pub transient: TransientPassthroughState, + pub last_client_access_at_ms: u64, + pub last_effective_manifest_host: Option, + pub origin_refresh: OriginRefreshState, + pub render_policy: RenderPolicy, + pub last_rendered_manifest: Option, + pub longest_rendered_playlist_duration_ms: u64, + pub initial_prefetch_gap_segments: usize, + pub segment_prefetch_queue: SegmentPrefetchQueue, + pub active_segment_fetches: usize, + pub segment_fetch_notifiers: HashMap>, + pub origin_request_headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub activity: HlsSessionActivity, + pub origin_epoch: u64, + pub origin_seq_highwater: Option, + pub proxy_next_seq: Option, + pub origin_to_proxy: HashMap, + pub discontinuity_sequence: u64, + pub transient_discontinuity_sequence: Option, + pub pending_handoff_discontinuity_sequence: Option, + pub pending_origin_epoch_handoff: bool, + pub segments: BTreeMap, + pub active_map_fetches: usize, + pub maps: BTreeMap, + pub origin_map_to_proxy: HashMap, + pub next_proxy_map_id: u64, + pub publishable_origin_head_proxy_seq: Option, + pub publishable_origin_tail_proxy_seq: Option, + pub origin_version: Option, + pub target_duration: Option, + pub independent_segments: bool, + pub fresh_manifest_commit_required: Option, + pub segment_failure_tracker: HlsSegmentFailureTracker, + pub manifest_temporary_failure_tracker: HlsManifestTemporaryFailureTracker, + pub manifest_acceptance: HlsManifestAcceptanceState, + gc_marked_for_removal: bool, +} + +impl HlsSession { + pub fn new(key: HlsSessionKey, reverse_proxy_rewrite_secret: &[u8], now_ms: u64) -> Self { + let origin_source = HlsOriginSource::from_session_key(&key); + Self::new_with_origin_source(key, origin_source, reverse_proxy_rewrite_secret, now_ms) + } + + pub fn new_with_origin_source( + key: HlsSessionKey, + origin_source: HlsOriginSource, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> Self { + let proxy_session_id = build_proxy_session_id(&key, reverse_proxy_rewrite_secret); + Self { + key, + proxy_session_id, + origin_source, + origin_account_binding: None, + origin_account_io_lease: None, + origin_account_rebind: HlsOriginAccountRebindState::default(), + effective_origin_acquire_policy: None, + mode: HlsSessionMode::NormalCacheTimeline, + transient: TransientPassthroughState::default(), + last_client_access_at_ms: now_ms, + last_effective_manifest_host: None, + origin_refresh: OriginRefreshState::default(), + render_policy: RenderPolicy::default(), + last_rendered_manifest: None, + longest_rendered_playlist_duration_ms: 0, + initial_prefetch_gap_segments: 0, + segment_prefetch_queue: SegmentPrefetchQueue::default(), + active_segment_fetches: 0, + segment_fetch_notifiers: HashMap::new(), + origin_request_headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + activity: HlsSessionActivity::default(), + origin_epoch: 0, + origin_seq_highwater: None, + proxy_next_seq: None, + origin_to_proxy: HashMap::new(), + discontinuity_sequence: 0, + transient_discontinuity_sequence: None, + pending_handoff_discontinuity_sequence: None, + pending_origin_epoch_handoff: false, + active_map_fetches: 0, + segments: BTreeMap::new(), + maps: BTreeMap::new(), + origin_map_to_proxy: HashMap::new(), + next_proxy_map_id: 0, + publishable_origin_head_proxy_seq: None, + publishable_origin_tail_proxy_seq: None, + origin_version: None, + target_duration: None, + independent_segments: false, + fresh_manifest_commit_required: None, + segment_failure_tracker: HlsSegmentFailureTracker::default(), + manifest_temporary_failure_tracker: HlsManifestTemporaryFailureTracker::default(), + manifest_acceptance: HlsManifestAcceptanceState::default(), + gc_marked_for_removal: false, + } + } + + pub fn mark_pending_handoff_discontinuity(&mut self, discontinuity_sequence: u64) { + self.pending_handoff_discontinuity_sequence = Some(discontinuity_sequence); + } + + pub fn mark_pending_origin_epoch_handoff_discontinuity(&mut self, discontinuity_sequence: u64) { + if self.pending_handoff_discontinuity_sequence.is_none() { + self.pending_handoff_discontinuity_sequence = Some(discontinuity_sequence); + } + self.pending_origin_epoch_handoff = true; + } + + pub fn take_pending_handoff_discontinuity_sequence(&mut self) -> Option { + self.pending_handoff_discontinuity_sequence.take() + } + + pub fn mark_for_gc_removal(&mut self) { self.gc_marked_for_removal = true; } + + pub fn clear_gc_removal_mark(&mut self) { self.gc_marked_for_removal = false; } + + pub fn is_gc_marked_for_removal(&self) -> bool { self.gc_marked_for_removal } + + pub fn replace_origin_account_binding(&mut self, binding: Option) { + let binding_changed = match (&self.origin_account_binding, &binding) { + (Some(current), Some(next)) => { + current.input_name != next.input_name + || current.account_name != next.account_name + || current.session_owner != next.session_owner + || current.generation != next.generation + } + (None, None) => false, + _ => true, + }; + if binding_changed { + self.origin_provider_session_headers.clear(); + } + self.origin_account_binding = binding; + } + + pub fn segment_temporary_failure_threshold(&self, fallback_threshold: u32) -> u32 { + let initial_prefetch_gap_segments = + u32::try_from(self.initial_prefetch_gap_segments).unwrap_or(u32::MAX.saturating_sub(3)); + fallback_threshold.max(3_u32.saturating_add(initial_prefetch_gap_segments)) + } + + pub fn record_successful_segment_fetch(&mut self) -> Option { + if self.segment_failure_tracker.consecutive_temporary_failures == 0 { + return None; + } + let previous = self.segment_failure_tracker.consecutive_temporary_failures; + self.segment_failure_tracker = HlsSegmentFailureTracker::default(); + Some(previous) + } + + pub fn record_temporary_segment_fetch_failure( + &mut self, + now_ms: u64, + object: HlsSegmentFailureObject, + threshold: u32, + ) -> HlsSegmentFailureTransition { + let failures = self.segment_failure_tracker.consecutive_temporary_failures.saturating_add(1); + self.segment_failure_tracker.consecutive_temporary_failures = failures; + self.segment_failure_tracker.last_failure_at_ms = Some(now_ms); + self.segment_failure_tracker.last_failed_object = Some(object); + + if failures >= threshold { + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } + } else { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } + } + } + + pub fn record_successful_manifest_fetch(&mut self) -> Option { + self.fresh_manifest_commit_required = None; + if self.manifest_temporary_failure_tracker.consecutive_temporary_failures == 0 { + return None; + } + let previous = self.manifest_temporary_failure_tracker.consecutive_temporary_failures; + self.manifest_temporary_failure_tracker = HlsManifestTemporaryFailureTracker::default(); + Some(previous) + } + + pub fn require_fresh_manifest_commit(&mut self, reason: HlsFreshManifestRequiredReason) { + self.fresh_manifest_commit_required = Some(reason); + } + + pub fn record_temporary_manifest_fetch_failure( + &mut self, + now_ms: u64, + kind: HlsManifestTemporaryFailureKind, + threshold: u32, + ) -> HlsManifestTemporaryFailureTransition { + let failures = self.manifest_temporary_failure_tracker.consecutive_temporary_failures.saturating_add(1); + self.manifest_temporary_failure_tracker.consecutive_temporary_failures = failures; + self.manifest_temporary_failure_tracker.last_failure_at_ms = Some(now_ms); + self.manifest_temporary_failure_tracker.last_failure_kind = Some(kind); + + if failures >= threshold { + HlsManifestTemporaryFailureTransition::BecameChannelUnavailable { failures, threshold } + } else { + HlsManifestTemporaryFailureTransition::StillRetryable { failures, threshold } + } + } + + pub fn account_overlap_timing(&self) -> HlsAccountOverlapTiming { + HlsAccountOverlapTiming::from_target_duration_secs(self.target_duration.map(u64::from)) + } + + pub fn account_binding_protection(&self, now_ms: u64) -> HlsAccountBindingProtection { + classify_account_binding_protection( + self.activity.last_authorized_media_at_ms, + now_ms, + self.account_overlap_timing(), + ) + } + + pub fn should_refresh_origin_reservation(&self, now_ms: u64) -> bool { + !matches!(self.account_binding_protection(now_ms), HlsAccountBindingProtection::Expired) + || self.activity.active_origin_work_count > 0 + } + + pub fn reconcile_effective_origin_acquire_policy( + &mut self, + candidate: Option, + now_ms: u64, + ) { + let Some(candidate) = candidate.map(|policy| policy.with_updated_at(now_ms)) else { + if self.effective_origin_policy_downgrade_allowed(now_ms) { + self.effective_origin_acquire_policy = None; + } + return; + }; + + let Some(state) = self.effective_origin_acquire_policy else { + self.effective_origin_acquire_policy = Some(HlsEffectiveOriginAcquirePolicyState::new(candidate, now_ms)); + return; + }; + + if candidate.has_same_rank_as(state.current_policy) { + self.effective_origin_acquire_policy = Some(HlsEffectiveOriginAcquirePolicyState { + current_policy: state.current_policy, + last_supported_at_ms: now_ms, + }); + } else if candidate.is_better_than(state.current_policy) + || self.effective_origin_policy_downgrade_allowed(now_ms) + { + self.effective_origin_acquire_policy = Some(HlsEffectiveOriginAcquirePolicyState::new(candidate, now_ms)); + } + } + + fn effective_origin_policy_downgrade_allowed(&self, now_ms: u64) -> bool { + let Some(state) = self.effective_origin_acquire_policy else { + return true; + }; + self.target_duration.is_none_or(|target_duration| { + let guard_ms = u64::from(target_duration).saturating_mul(1_000); + now_ms.saturating_sub(state.last_supported_at_ms) >= guard_ms + }) + } + + pub fn effective_origin_acquire_policy_or_default(&self) -> HlsEffectiveOriginAcquirePolicy { + self.effective_origin_acquire_policy + .map_or_else(HlsEffectiveOriginAcquirePolicy::fallback, |state| state.current_policy) + } + + pub fn mark_authorized_manifest_access(&mut self, now_ms: u64) { + self.activity.last_authorized_manifest_at_ms = Some(now_ms); + self.last_client_access_at_ms = now_ms; + } + + pub fn mark_authorized_media_access(&mut self, now_ms: u64) { + self.activity.last_authorized_media_at_ms = Some(now_ms); + self.last_client_access_at_ms = now_ms; + } + + pub fn idle_expiry_due_at_ms(&self, session_idle_timeout_ms: u64) -> u64 { + self.last_client_access_at_ms.saturating_add(session_idle_timeout_ms) + } + + pub fn can_expire_idle_session(&self, now_ms: u64, session_idle_timeout_ms: u64) -> bool { + if self.idle_expiry_due_at_ms(session_idle_timeout_ms) > now_ms { + return false; + } + self.activity.active_origin_work_count == 0 + && self.active_segment_fetches == 0 + && self.active_map_fetches == 0 + && !self.origin_refresh.in_flight + && self.segment_prefetch_queue.is_empty() + && !self.segments.values().any(|segment| { + segment.access.active_readers() > 0 || matches!(segment.status, SegmentCacheStatus::Fetching { .. }) + }) + && !self.maps.values().any(|map| { + map.access.active_readers() > 0 + || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) + }) + && !self.transient.has_active_resource_readers() + } + + pub fn initial_manifest_commit_work_pending(&self) -> bool { + self.origin_refresh.in_flight + || self.active_segment_fetches > 0 + || self.active_map_fetches > 0 + || self.segments.values().any(|segment| { + matches!( + segment.status, + SegmentCacheStatus::Queued { + priority: SegmentFetchPriority::Demand | SegmentFetchPriority::RenderWindow, + .. + } | SegmentCacheStatus::Fetching { + priority: SegmentFetchPriority::Demand | SegmentFetchPriority::RenderWindow, + .. + } + ) + }) + || self + .maps + .values() + .any(|map| matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. })) + } + + pub fn start_origin_work(&mut self) -> u64 { + self.activity.active_origin_work_count = self.activity.active_origin_work_count.saturating_add(1); + self.activity.origin_work_generation + } + + pub fn finish_origin_work(&mut self, started_generation: u64) -> bool { + self.activity.active_origin_work_count = self.activity.active_origin_work_count.saturating_sub(1); + started_generation == self.activity.origin_work_generation + } + + pub fn invalidate_queued_origin_work(&mut self) { + self.activity.origin_work_generation = self.activity.origin_work_generation.saturating_add(1); + let queued_segment_seqs = self.segment_prefetch_queue.proxy_seqs(); + for proxy_seq in queued_segment_seqs { + self.segment_prefetch_queue.remove(proxy_seq); + if let Some(segment) = self.segments.get_mut(&proxy_seq) { + if matches!(segment.status, SegmentCacheStatus::Queued { .. }) { + segment.status = SegmentCacheStatus::Discovered; + } + } + } + for map in self.maps.values_mut() { + if matches!(map.status, MapCacheStatus::Queued { .. }) { + map.status = MapCacheStatus::Discovered; + } + } + } + + pub fn referenced_map_ids(&self) -> Vec { + let mut map_ids = self.segments.values().filter_map(|segment| segment.map_ref).collect::>(); + map_ids.sort_unstable(); + map_ids.dedup(); + map_ids + } +} + +impl fmt::Debug for HlsSession { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HlsSession") + .field("key", &self.key) + .field("proxy_session_id", &self.proxy_session_id) + .field("origin_source", &self.origin_source) + .field("origin_account_binding", &self.origin_account_binding) + .field("origin_account_io_lease", &self.origin_account_io_lease) + .field("origin_account_rebind", &self.origin_account_rebind) + .field("effective_origin_acquire_policy", &self.effective_origin_acquire_policy) + .field("mode", &self.mode) + .field("transient", &self.transient) + .field("last_client_access_at_ms", &self.last_client_access_at_ms) + .field("last_effective_manifest_host", &self.last_effective_manifest_host) + .field("origin_refresh", &self.origin_refresh) + .field("render_policy", &self.render_policy) + .field("last_rendered_manifest", &self.last_rendered_manifest) + .field("longest_rendered_playlist_duration_ms", &self.longest_rendered_playlist_duration_ms) + .field("initial_prefetch_gap_segments", &self.initial_prefetch_gap_segments) + .field("segment_prefetch_queue_len", &self.segment_prefetch_queue.len()) + .field("active_segment_fetches", &self.active_segment_fetches) + .field("segment_fetch_notifiers_len", &self.segment_fetch_notifiers.len()) + .field("origin_request_headers_len", &self.origin_request_headers.len()) + .field("origin_provider_session_headers_len", &self.origin_provider_session_headers.len()) + .field("activity", &self.activity) + .field("origin_epoch", &self.origin_epoch) + .field("origin_seq_highwater", &self.origin_seq_highwater) + .field("proxy_next_seq", &self.proxy_next_seq) + .field("origin_to_proxy_len", &self.origin_to_proxy.len()) + .field("discontinuity_sequence", &self.discontinuity_sequence) + .field("transient_discontinuity_sequence", &self.transient_discontinuity_sequence) + .field("pending_handoff_discontinuity_sequence", &self.pending_handoff_discontinuity_sequence) + .field("pending_origin_epoch_handoff", &self.pending_origin_epoch_handoff) + .field("active_map_fetches", &self.active_map_fetches) + .field("segments_len", &self.segments.len()) + .field("maps_len", &self.maps.len()) + .field("origin_map_to_proxy_len", &self.origin_map_to_proxy.len()) + .field("next_proxy_map_id", &self.next_proxy_map_id) + .field("publishable_origin_head_proxy_seq", &self.publishable_origin_head_proxy_seq) + .field("publishable_origin_tail_proxy_seq", &self.publishable_origin_tail_proxy_seq) + .field("origin_version", &self.origin_version) + .field("target_duration", &self.target_duration) + .field("independent_segments", &self.independent_segments) + .field("fresh_manifest_commit_required", &self.fresh_manifest_commit_required) + .field("segment_failure_tracker", &self.segment_failure_tracker) + .field("manifest_temporary_failure_tracker", &self.manifest_temporary_failure_tracker) + .field("manifest_acceptance", &self.manifest_acceptance) + .field("gc_marked_for_removal", &self.gc_marked_for_removal) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::HlsSession; + use crate::api::model::{ + ConnectionKind, HlsAccountBindingProtection, HlsEffectiveOriginAcquirePolicy, HlsFreshManifestRequiredReason, + HlsManifestTemporaryFailureKind, HlsManifestTemporaryFailureTransition, HlsSegmentFailureObject, + HlsSegmentFailureTransition, HlsSessionKey, + }; + use axum::http::StatusCode; + + fn origin_policy(connection_kind: ConnectionKind, priority: i8) -> HlsEffectiveOriginAcquirePolicy { + HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, 0) + } + + #[test] + fn manifest_access_updates_only_manifest_activity() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + session.mark_authorized_manifest_access(1_000); + + assert_eq!(session.activity.last_authorized_manifest_at_ms, Some(1_000)); + assert_eq!(session.activity.last_authorized_media_at_ms, None); + } + + #[test] + fn media_access_updates_media_activity_and_protection() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(12); + + session.mark_authorized_media_access(1_000); + + assert_eq!(session.activity.last_authorized_media_at_ms, Some(1_000)); + assert_eq!( + session.account_binding_protection(5_000), + HlsAccountBindingProtection::HardActive { until_ms: 13_000 } + ); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); + assert_eq!(session.account_overlap_timing().hard_active_window_ms, 12_000); + assert_eq!(session.account_overlap_timing().soft_active_window_ms, 24_000); + } + + #[test] + fn missing_target_duration_uses_account_overlap_fallback_windows() { + let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!(session.account_overlap_timing().target_duration_ms, 15_000); + assert_eq!(session.account_overlap_timing().hard_active_window_ms, 15_000); + assert_eq!(session.account_overlap_timing().soft_active_window_ms, 30_000); + } + + #[test] + fn no_media_activity_is_not_expired_protection() { + let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!(session.account_binding_protection(1_000), HlsAccountBindingProtection::NoMediaYet); + assert!(session.should_refresh_origin_reservation(1_000)); + } + + #[test] + fn idle_session_can_expire_at_exact_idle_boundary() { + let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert!(session.can_expire_idle_session(1_000, 1_000)); + } + + #[test] + fn active_origin_work_blocks_idle_session_expiry() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.start_origin_work(); + + assert!(!session.can_expire_idle_session(1_000, 1_000)); + } + + #[test] + fn origin_work_generation_invalidates_queued_work() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let started_generation = session.start_origin_work(); + + session.invalidate_queued_origin_work(); + + assert!(!session.finish_origin_work(started_generation)); + assert_eq!(session.activity.active_origin_work_count, 0); + assert_eq!(session.activity.origin_work_generation, 1); + } + + #[test] + fn temporary_segment_failures_reach_threshold() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!( + session.record_temporary_segment_fetch_failure( + 1_000, + HlsSegmentFailureObject::Normal { proxy_seq: 1, origin_seq: 10 }, + 2, + ), + HlsSegmentFailureTransition::StillRetryable { failures: 1, threshold: 2 } + ); + assert_eq!( + session.record_temporary_segment_fetch_failure( + 2_000, + HlsSegmentFailureObject::Normal { proxy_seq: 2, origin_seq: 11 }, + 2, + ), + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures: 2, threshold: 2 } + ); + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 2); + assert_eq!(session.segment_failure_tracker.last_failure_at_ms, Some(2_000)); + assert_eq!( + session.segment_failure_tracker.last_failed_object, + Some(HlsSegmentFailureObject::Normal { proxy_seq: 2, origin_seq: 11 }) + ); + } + + #[test] + fn successful_segment_fetch_resets_temporary_failure_counter() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let transition = session.record_temporary_segment_fetch_failure( + 1_000, + HlsSegmentFailureObject::Transient { resource_id: "abc".to_string() }, + 3, + ); + + assert_eq!(transition, HlsSegmentFailureTransition::StillRetryable { failures: 1, threshold: 3 }); + assert_eq!(session.record_successful_segment_fetch(), Some(1)); + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); + assert!(session.segment_failure_tracker.last_failure_at_ms.is_none()); + assert!(session.segment_failure_tracker.last_failed_object.is_none()); + } + + #[test] + fn temporary_manifest_failures_reach_threshold() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!( + session.record_temporary_manifest_fetch_failure(1_000, HlsManifestTemporaryFailureKind::Timeout, 2), + HlsManifestTemporaryFailureTransition::StillRetryable { failures: 1, threshold: 2 } + ); + assert_eq!( + session.record_temporary_manifest_fetch_failure( + 2_000, + HlsManifestTemporaryFailureKind::RetryableStatus { status: StatusCode::TOO_MANY_REQUESTS }, + 2, + ), + HlsManifestTemporaryFailureTransition::BecameChannelUnavailable { failures: 2, threshold: 2 } + ); + assert_eq!(session.manifest_temporary_failure_tracker.consecutive_temporary_failures, 2); + assert_eq!(session.manifest_temporary_failure_tracker.last_failure_at_ms, Some(2_000)); + assert_eq!( + session.manifest_temporary_failure_tracker.last_failure_kind, + Some(HlsManifestTemporaryFailureKind::RetryableStatus { status: StatusCode::TOO_MANY_REQUESTS }) + ); + } + + #[test] + fn successful_manifest_fetch_resets_temporary_failure_counter() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + let transition = + session.record_temporary_manifest_fetch_failure(1_000, HlsManifestTemporaryFailureKind::Timeout, 3); + + assert_eq!(transition, HlsManifestTemporaryFailureTransition::StillRetryable { failures: 1, threshold: 3 }); + assert_eq!(session.record_successful_manifest_fetch(), Some(1)); + assert_eq!(session.fresh_manifest_commit_required, None); + assert_eq!(session.manifest_temporary_failure_tracker.consecutive_temporary_failures, 0); + assert!(session.manifest_temporary_failure_tracker.last_failure_at_ms.is_none()); + assert!(session.manifest_temporary_failure_tracker.last_failure_kind.is_none()); + } + + #[test] + fn effective_origin_policy_upgrade_is_applied_immediately() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(12); + + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Soft, -20)), 1_000); + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Normal, 50)), 1_001); + let normal_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(normal_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(normal_policy.priority, 50); + + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Normal, -5)), 1_002); + let upgraded_priority = session.effective_origin_acquire_policy_or_default(); + assert_eq!(upgraded_priority.connection_kind, ConnectionKind::Normal); + assert_eq!(upgraded_priority.priority, -5); + } + + #[test] + fn effective_origin_policy_downgrade_waits_for_session_target_duration() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(12); + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Normal, -5)), 1_000); + + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Soft, -20)), 5_000); + let protected_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(protected_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(protected_policy.priority, -5); + + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Soft, -20)), 13_000); + let downgraded_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(downgraded_policy.connection_kind, ConnectionKind::Soft); + assert_eq!(downgraded_policy.priority, -20); + } + + #[test] + fn effective_origin_policy_downgrades_without_delay_when_target_duration_is_unknown() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Normal, -5)), 1_000); + + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Soft, -20)), 1_001); + + let policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(policy.connection_kind, ConnectionKind::Soft); + assert_eq!(policy.priority, -20); + } + + #[test] + fn effective_origin_policy_clear_waits_for_session_target_duration() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(10); + session.reconcile_effective_origin_acquire_policy(Some(origin_policy(ConnectionKind::Normal, -5)), 1_000); + + session.reconcile_effective_origin_acquire_policy(None, 5_000); + assert!(session.effective_origin_acquire_policy.is_some()); + let protected_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(protected_policy.priority, -5); + + session.reconcile_effective_origin_acquire_policy(None, 11_000); + assert!(session.effective_origin_acquire_policy.is_none()); + let fallback = session.effective_origin_acquire_policy_or_default(); + assert_eq!(fallback.connection_kind, ConnectionKind::Normal); + assert_eq!(fallback.priority, 0); + } +} diff --git a/backend/src/api/model/hls_cache/session_store.rs b/backend/src/api/model/hls_cache/session_store.rs new file mode 100644 index 000000000..5fe990bad --- /dev/null +++ b/backend/src/api/model/hls_cache/session_store.rs @@ -0,0 +1,349 @@ +use super::{HlsOriginSource, HlsSession, HlsSessionKey, ProxySessionId}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; + +pub type HlsSessionHandle = Arc>; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsSessionStoreOutcome { + Created, + Reused, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsExpiredSessionReason { + SessionIdleTimeout, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsExpiredSessionMarker { + pub proxy_session_id: ProxySessionId, + pub session_key: HlsSessionKey, + pub username: Option, + pub expired_at_ms: u64, + pub reason: HlsExpiredSessionReason, +} + +/// In-memory lookup store for HLS sessions by stable key and public proxy ID. +#[derive(Default)] +pub struct HlsSessionStore { + indexes: RwLock, +} + +#[derive(Default)] +struct SessionIndexes { + by_key: HashMap, + by_proxy_session_id: HashMap, + expired_by_proxy_session_id: HashMap, +} + +impl HlsSessionStore { + pub fn new() -> Self { Self::default() } + + pub async fn get_by_key(&self, key: &HlsSessionKey) -> Option { + self.indexes.read().await.by_key.get(key).map(Arc::clone) + } + + pub async fn get_by_proxy_session_id(&self, proxy_session_id: &ProxySessionId) -> Option { + self.indexes.read().await.by_proxy_session_id.get(proxy_session_id).map(Arc::clone) + } + + pub async fn list_sessions(&self) -> Vec { + self.indexes.read().await.by_key.values().map(Arc::clone).collect() + } + + pub async fn get_or_create_session( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> HlsSessionHandle { + self.get_or_create_session_with_outcome(key, reverse_proxy_rewrite_secret, now_ms).await.0 + } + + pub async fn get_or_create_session_with_outcome( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let origin_source = HlsOriginSource::from_session_key(&key); + self.get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) + .await + } + + pub async fn get_or_create_session_with_source_and_outcome( + &self, + key: HlsSessionKey, + origin_source: HlsOriginSource, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let mut indexes = self.indexes.write().await; + if let Some(session) = indexes.by_key.get(&key) { + return (Arc::clone(session), HlsSessionStoreOutcome::Reused); + } + + let session = + HlsSession::new_with_origin_source(key.clone(), origin_source, reverse_proxy_rewrite_secret, now_ms); + let proxy_session_id = session.proxy_session_id.clone(); + let session = Arc::new(RwLock::new(session)); + indexes.by_key.insert(key, Arc::clone(&session)); + indexes.expired_by_proxy_session_id.remove(&proxy_session_id); + indexes.by_proxy_session_id.insert(proxy_session_id, Arc::clone(&session)); + (session, HlsSessionStoreOutcome::Created) + } + + pub async fn remove_session( + &self, + key: &HlsSessionKey, + _proxy_session_id: &ProxySessionId, + ) -> Option { + let mut indexes = self.indexes.write().await; + let (proxy_session_id, removed) = remove_indexed_session(&mut indexes, key)?; + indexes.expired_by_proxy_session_id.remove(&proxy_session_id); + Some(removed) + } + + pub async fn remove_session_marking_expired( + &self, + key: &HlsSessionKey, + _proxy_session_id: &ProxySessionId, + now_ms: u64, + reason: HlsExpiredSessionReason, + username: Option, + ) -> Option { + let mut indexes = self.indexes.write().await; + let (proxy_session_id, removed) = remove_indexed_session(&mut indexes, key)?; + indexes.expired_by_proxy_session_id.insert( + proxy_session_id.clone(), + HlsExpiredSessionMarker { + proxy_session_id: proxy_session_id.clone(), + session_key: key.clone(), + username, + expired_at_ms: now_ms, + reason, + }, + ); + Some(removed) + } + + pub async fn expired_session_marker( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + retention_ms: u64, + ) -> Option { + let mut indexes = self.indexes.write().await; + let marker = indexes.expired_by_proxy_session_id.get(proxy_session_id)?; + if marker.expired_at_ms.saturating_add(retention_ms) <= now_ms { + indexes.expired_by_proxy_session_id.remove(proxy_session_id); + return None; + } + Some(marker.clone()) + } + + pub async fn update_expired_session_marker_username( + &self, + proxy_session_id: &ProxySessionId, + username: Option, + ) { + let Some(username) = username else { + return; + }; + if let Some(marker) = self.indexes.write().await.expired_by_proxy_session_id.get_mut(proxy_session_id) { + marker.username.get_or_insert(username); + } + } + + pub async fn clear(&self) { + let mut indexes = self.indexes.write().await; + indexes.by_key.clear(); + indexes.by_proxy_session_id.clear(); + indexes.expired_by_proxy_session_id.clear(); + } + + #[cfg(test)] + async fn proxy_session_index_len(&self) -> usize { self.indexes.read().await.by_proxy_session_id.len() } + + #[cfg(test)] + pub async fn len(&self) -> usize { self.indexes.read().await.by_key.len() } + + #[cfg(test)] + pub async fn is_empty(&self) -> bool { self.indexes.read().await.by_key.is_empty() } +} + +fn remove_indexed_session( + indexes: &mut SessionIndexes, + key: &HlsSessionKey, +) -> Option<(ProxySessionId, HlsSessionHandle)> { + let session = indexes.by_key.get(key)?; + let proxy_session_id = indexes + .by_proxy_session_id + .iter() + .find_map(|(proxy_session_id, indexed)| Arc::ptr_eq(session, indexed).then(|| proxy_session_id.clone()))?; + let removed = indexes.by_key.remove(key)?; + indexes.by_proxy_session_id.remove(&proxy_session_id); + Some((proxy_session_id, removed)) +} + +#[cfg(test)] +mod tests { + use super::{HlsExpiredSessionReason, HlsSessionStore}; + use crate::api::model::{HlsSessionKey, ProxySessionId}; + use std::sync::Arc; + + #[tokio::test] + async fn get_or_create_session_reuses_existing_session_for_same_key() { + let store = HlsSessionStore::new(); + let key = HlsSessionKey::new(1, "12345"); + + let first = store.get_or_create_session(key.clone(), b"0011223344556677", 100).await; + let second = store.get_or_create_session(key, b"0011223344556677", 200).await; + + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(first.read().await.last_client_access_at_ms, 100); + } + + #[tokio::test] + async fn proxy_session_id_lookup_finds_created_session() { + let store = HlsSessionStore::new(); + let key = HlsSessionKey::new(1, "12345"); + let created = store.get_or_create_session(key, b"0011223344556677", 100).await; + let proxy_session_id = created.read().await.proxy_session_id.clone(); + + let found = store + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("session should be indexed by proxy_session_id"); + + assert!(Arc::ptr_eq(&created, &found)); + } + + #[tokio::test] + async fn remove_session_marking_expired_retains_marker_until_recreated_or_expired() { + let store = HlsSessionStore::new(); + let key = HlsSessionKey::new(1, "12345"); + let created = store.get_or_create_session(key.clone(), b"0011223344556677", 100).await; + let proxy_session_id = created.read().await.proxy_session_id.clone(); + + let removed = store + .remove_session_marking_expired( + &key, + &proxy_session_id, + 1_000, + HlsExpiredSessionReason::SessionIdleTimeout, + Some("viewer".to_string()), + ) + .await; + assert!(removed.is_some()); + + let marker = store + .expired_session_marker(&proxy_session_id, 1_500, 10_000) + .await + .expect("expired marker should remain within retention"); + assert_eq!(marker.username.as_deref(), Some("viewer")); + assert_eq!(marker.reason, HlsExpiredSessionReason::SessionIdleTimeout); + + let expired = store.expired_session_marker(&proxy_session_id, 11_000, 10_000).await; + assert!(expired.is_none(), "expired marker should be pruned after retention"); + + let recreated = store.get_or_create_session(key, b"0011223344556677", 12_000).await; + assert_eq!(recreated.read().await.proxy_session_id, proxy_session_id); + assert!(store.expired_session_marker(&proxy_session_id, 12_100, 10_000).await.is_none()); + } + + #[tokio::test] + async fn remove_session_cleans_indexes_by_actual_session_id() { + let store = HlsSessionStore::new(); + let key = HlsSessionKey::new(1, "12345"); + let created = store.get_or_create_session(key.clone(), b"0011223344556677", 100).await; + let proxy_session_id = created.read().await.proxy_session_id.clone(); + let stale_proxy_session_id = ProxySessionId("stale".to_string()); + + let removed = store.remove_session(&key, &stale_proxy_session_id).await; + + assert!(removed.is_some()); + assert!(store.get_by_proxy_session_id(&proxy_session_id).await.is_none()); + assert!(store.get_by_proxy_session_id(&stale_proxy_session_id).await.is_none()); + } + + #[tokio::test] + async fn remove_session_marking_expired_uses_actual_session_id_for_marker() { + let store = HlsSessionStore::new(); + let key = HlsSessionKey::new(1, "12345"); + let created = store.get_or_create_session(key.clone(), b"0011223344556677", 100).await; + let proxy_session_id = created.read().await.proxy_session_id.clone(); + let stale_proxy_session_id = ProxySessionId("stale".to_string()); + + let removed = store + .remove_session_marking_expired( + &key, + &stale_proxy_session_id, + 1_000, + HlsExpiredSessionReason::SessionIdleTimeout, + None, + ) + .await; + + assert!(removed.is_some()); + assert!(store.expired_session_marker(&proxy_session_id, 1_500, 10_000).await.is_some()); + assert!(store.expired_session_marker(&stale_proxy_session_id, 1_500, 10_000).await.is_none()); + } + + #[tokio::test] + async fn parallel_get_or_create_session_creates_single_index_entry() { + let store = Arc::new(HlsSessionStore::new()); + let key = HlsSessionKey::new(1, "12345"); + let mut tasks = Vec::new(); + + for now_ms in 100..108 { + let store = Arc::clone(&store); + let key = key.clone(); + tasks + .push(tokio::spawn(async move { store.get_or_create_session(key, b"0011223344556677", now_ms).await })); + } + + let first = tasks.remove(0).await.expect("task should not panic"); + for task in tasks { + let session = task.await.expect("task should not panic"); + assert!(Arc::ptr_eq(&first, &session)); + } + + assert_eq!(store.proxy_session_index_len().await, 1); + } + + #[tokio::test] + async fn removal_does_not_remove_a_concurrently_recreated_session() { + let store = Arc::new(HlsSessionStore::new()); + let key = HlsSessionKey::new(1, "12345"); + let old = store.get_or_create_session(key.clone(), b"0011223344556677", 100).await; + let proxy_session_id = old.read().await.proxy_session_id.clone(); + let old_guard = old.write().await; + let removal_store = Arc::clone(&store); + let removal_key = key.clone(); + let removal_proxy_session_id = proxy_session_id.clone(); + let removal = tokio::spawn(async move { + removal_store + .remove_session_marking_expired( + &removal_key, + &removal_proxy_session_id, + 1_000, + HlsExpiredSessionReason::SessionIdleTimeout, + None, + ) + .await + }); + + while store.get_by_key(&key).await.is_some() { + tokio::task::yield_now().await; + } + let recreated = store.get_or_create_session(key, b"0011223344556677", 1_001).await; + drop(old_guard); + assert!(matches!(removal.await, Ok(Some(_)))); + + let indexed = store.get_by_proxy_session_id(&proxy_session_id).await; + assert!(indexed.as_ref().is_some_and(|session| Arc::ptr_eq(session, &recreated))); + assert!(store.expired_session_marker(&proxy_session_id, 1_002, 10_000).await.is_none()); + } +} diff --git a/backend/src/api/model/hls_cache/timeline.rs b/backend/src/api/model/hls_cache/timeline.rs new file mode 100644 index 000000000..e54e5720f --- /dev/null +++ b/backend/src/api/model/hls_cache/timeline.rs @@ -0,0 +1,889 @@ +use super::{ + safe_proxy_session_id, HlsSession, MapEntry, OriginMapKey, ProxyMapId, ProxySessionId, SegmentCacheKey, + SegmentFetchPriority, +}; +use crate::processing::parser::hls::origin_manifest::{ + ParsedByteRange, ParsedOriginManifest, ParsedOriginMap, ParsedOriginSegment, +}; +use axum::http::StatusCode; +use log::info; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + sync::{ + atomic::{AtomicU32, AtomicU64, Ordering}, + Arc, + }, +}; +use url::Url; + +const SEGMENT_EXTENSIONS: &[&str] = &["ts", "mp4", "m4s", "m4v"]; +const MAP_EXTENSIONS: &[&str] = &["mp4", "m4s", "m4v"]; +pub const HLS_PROVISIONING_ORIGIN_EPOCH: u64 = u64::MAX; +pub const HLS_PROVISIONING_GAP_ORIGIN_EPOCH: u64 = u64::MAX - 1; +pub const HLS_PROVISIONING_TARGET_DURATION_SECS: u32 = 2; +pub const HLS_PROVISIONING_SEGMENT_DURATION_MS: u64 = 2_000; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub struct OriginSegmentKey { + pub origin_epoch: u64, + pub origin_seq: u64, +} + +/// Volatile concrete origin URL for one segment download. +/// +/// The URL is resolved against the final fetched manifest URL and may include a redirect/CDN host. Use it only as a +/// fetch target or sanitized diagnostics; stable timeline identity is `OriginSegmentKey`, and cache identity is the +/// proxy sequence based `SegmentCacheKey`. +#[derive(Clone, Eq, PartialEq)] +pub struct OriginSegmentFetchRef { + /// Concrete URL used to refetch the segment object. + /// + /// This starts from `ParsedOriginSegment::resolved_origin_url`, which is resolved against the final manifest URL + /// after redirects. It is a fetch target only; the normal timeline identity remains `OriginSegmentKey`. + pub resolved_origin_url: String, + pub byte_range: Option, + pub valid_until_ms: Option, +} + +impl OriginSegmentFetchRef { + pub fn is_valid_at(&self, now_ms: u64) -> bool { + self.valid_until_ms.is_none_or(|valid_until_ms| now_ms <= valid_until_ms) + } +} + +impl fmt::Debug for OriginSegmentFetchRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OriginSegmentFetchRef") + .field("resolved_origin_url", &"") + .field("byte_range", &self.byte_range) + .field("valid_until_ms", &self.valid_until_ms) + .finish() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum SegmentCacheStatus { + Discovered, + Queued { priority: SegmentFetchPriority, queued_at_ms: u64 }, + Fetching { priority: SegmentFetchPriority, started_at_ms: u64 }, + Ready { content_length: u64, ready_at_ms: u64 }, + FailedRetryable { failed_at_ms: u64, retry_after_ms: u64 }, + FailedPermanent { failed_at_ms: u64, status: Option }, + Expired, +} + +/// Mutable access counters shared with response streams without holding a session lock. +#[derive(Debug, Default)] +pub struct CacheAccessState { + active_readers: AtomicU32, + last_accessed_at_ms: AtomicU64, +} + +impl CacheAccessState { + pub fn new() -> Self { Self::default() } + + pub fn reader_started(&self, now_ms: u64) { + self.active_readers.fetch_add(1, Ordering::AcqRel); + self.last_accessed_at_ms.store(now_ms, Ordering::Release); + } + + pub fn reader_finished(&self) { + let mut current = self.active_readers.load(Ordering::Acquire); + while current > 0 { + match self.active_readers.compare_exchange_weak(current, current - 1, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return, + Err(next) => current = next, + } + } + } + + pub fn active_readers(&self) -> u32 { self.active_readers.load(Ordering::Acquire) } + + pub fn last_accessed_at_ms(&self) -> u64 { self.last_accessed_at_ms.load(Ordering::Acquire) } +} + +impl PartialEq for CacheAccessState { + fn eq(&self, other: &Self) -> bool { + self.active_readers() == other.active_readers() && self.last_accessed_at_ms() == other.last_accessed_at_ms() + } +} + +impl Eq for CacheAccessState {} + +pub fn default_content_type_for_segment_ext(extension: &str) -> &'static str { + match extension { + "ts" => "video/mp2t", + "mp4" | "m4v" | "m4s" => "video/mp4", + _ => "application/octet-stream", + } +} + +pub fn is_hls_provisioning_segment(entry: &SegmentEntry) -> bool { + entry.origin_key.origin_epoch == HLS_PROVISIONING_ORIGIN_EPOCH +} + +pub fn is_hls_provisioning_gap_segment(entry: &SegmentEntry) -> bool { + entry.origin_key.origin_epoch == HLS_PROVISIONING_GAP_ORIGIN_EPOCH +} + +#[derive(Clone, Eq, PartialEq)] +pub struct SegmentEntry { + pub origin_key: OriginSegmentKey, + pub proxy_seq: u64, + pub duration_ms: u64, + pub proxy_file_ext: String, + pub content_type: String, + pub cache_key: SegmentCacheKey, + pub discontinuity_before: bool, + pub program_date_time: Option, + pub daterange_tags_before: Vec, + pub origin_byte_range: Option, + pub map_ref: Option, + pub origin_fetch_ref: Option, + pub status: SegmentCacheStatus, + pub last_rendered_at_ms: Option, + pub access: Arc, +} + +impl fmt::Debug for SegmentEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentEntry") + .field("origin_key", &self.origin_key) + .field("proxy_seq", &self.proxy_seq) + .field("duration_ms", &self.duration_ms) + .field("proxy_file_ext", &self.proxy_file_ext) + .field("content_type", &self.content_type) + .field("cache_key", &self.cache_key) + .field("discontinuity_before", &self.discontinuity_before) + .field("program_date_time", &self.program_date_time) + .field("daterange_tags_before", &self.daterange_tags_before) + .field("origin_byte_range", &self.origin_byte_range) + .field("map_ref", &self.map_ref) + .field("origin_fetch_ref", &self.origin_fetch_ref) + .field("status", &self.status) + .field("last_rendered_at_ms", &self.last_rendered_at_ms) + .field("active_readers", &self.access.active_readers()) + .field("last_accessed_at_ms", &self.access.last_accessed_at_ms()) + .finish() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TimelineMapError { + UnsupportedSegmentExtension, + UnsupportedMapExtension, + ProxySequenceOverflow, + ProxyMapIdOverflow, +} + +#[derive(Clone)] +struct TimelineDraft { + proxy_session_id: ProxySessionId, + origin_epoch: u64, + origin_seq_highwater: Option, + proxy_next_seq: Option, + origin_to_proxy: HashMap, + discontinuity_sequence: u64, + pending_handoff_discontinuity_sequence: Option, + pending_origin_epoch_handoff: bool, + segments: BTreeMap, + maps: BTreeMap, + origin_map_to_proxy: HashMap, + next_proxy_map_id: u64, + publishable_origin_head_proxy_seq: Option, + publishable_origin_tail_proxy_seq: Option, + origin_version: Option, + target_duration: Option, + independent_segments: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum OriginEpochTransitionReason { + Rollover, + Handoff, +} + +impl From<&HlsSession> for TimelineDraft { + fn from(session: &HlsSession) -> Self { + Self { + proxy_session_id: session.proxy_session_id.clone(), + origin_epoch: session.origin_epoch, + origin_seq_highwater: session.origin_seq_highwater, + proxy_next_seq: session.proxy_next_seq, + origin_to_proxy: session.origin_to_proxy.clone(), + discontinuity_sequence: session.discontinuity_sequence, + pending_handoff_discontinuity_sequence: session.pending_handoff_discontinuity_sequence, + pending_origin_epoch_handoff: session.pending_origin_epoch_handoff, + segments: session.segments.clone(), + maps: session.maps.clone(), + origin_map_to_proxy: session.origin_map_to_proxy.clone(), + next_proxy_map_id: session.next_proxy_map_id, + publishable_origin_head_proxy_seq: session.publishable_origin_head_proxy_seq, + publishable_origin_tail_proxy_seq: session.publishable_origin_tail_proxy_seq, + origin_version: session.origin_version, + target_duration: session.target_duration, + independent_segments: session.independent_segments, + } + } +} + +impl HlsSession { + pub fn apply_origin_manifest(&mut self, manifest: &ParsedOriginManifest) -> Result<(), TimelineMapError> { + let mut draft = TimelineDraft::from(&*self); + draft.apply_manifest(manifest)?; + + self.origin_epoch = draft.origin_epoch; + self.origin_seq_highwater = draft.origin_seq_highwater; + self.proxy_next_seq = draft.proxy_next_seq; + self.origin_to_proxy = draft.origin_to_proxy; + self.discontinuity_sequence = draft.discontinuity_sequence; + self.pending_handoff_discontinuity_sequence = draft.pending_handoff_discontinuity_sequence; + self.pending_origin_epoch_handoff = draft.pending_origin_epoch_handoff; + self.segments = draft.segments; + self.maps = draft.maps; + self.origin_map_to_proxy = draft.origin_map_to_proxy; + self.next_proxy_map_id = draft.next_proxy_map_id; + self.publishable_origin_head_proxy_seq = draft.publishable_origin_head_proxy_seq; + self.publishable_origin_tail_proxy_seq = draft.publishable_origin_tail_proxy_seq; + self.origin_version = draft.origin_version; + self.target_duration = draft.target_duration; + self.independent_segments = draft.independent_segments; + + Ok(()) + } +} + +impl TimelineDraft { + fn apply_manifest(&mut self, manifest: &ParsedOriginManifest) -> Result<(), TimelineMapError> { + let handoff_discontinuity_sequence = self.pending_handoff_discontinuity_sequence.take(); + let handoff_publishable_head_proxy_seq = + handoff_discontinuity_sequence.and(self.publishable_origin_head_proxy_seq); + let origin_epoch_handoff = self.pending_origin_epoch_handoff; + self.pending_origin_epoch_handoff = false; + if self.segments.is_empty() { + self.discontinuity_sequence = manifest + .discontinuity_sequence + .unwrap_or(0) + .saturating_add(handoff_discontinuity_sequence.unwrap_or(0)); + } + self.origin_version = manifest.version; + self.target_duration = manifest.target_duration; + self.independent_segments = manifest.independent_segments; + self.apply_forward_jump(manifest)?; + let epoch_transition_discontinuity = self.apply_origin_epoch_transition_for_manifest(manifest, origin_epoch_handoff)?; + + let mut mark_handoff_discontinuity = + handoff_discontinuity_sequence.is_some() || epoch_transition_discontinuity; + let mut manifest_head_proxy_seq = None; + let mut manifest_tail_proxy_seq = None; + for parsed in &manifest.segments { + let proxy_seq = self.map_origin_segment(parsed, manifest, mark_handoff_discontinuity)?; + manifest_head_proxy_seq.get_or_insert(proxy_seq); + manifest_tail_proxy_seq = Some(proxy_seq); + mark_handoff_discontinuity = false; + } + + self.publishable_origin_head_proxy_seq = handoff_publishable_head_proxy_seq.or(manifest_head_proxy_seq); + self.publishable_origin_tail_proxy_seq = manifest_tail_proxy_seq; + Ok(()) + } + + fn apply_origin_epoch_transition_for_manifest( + &mut self, + manifest: &ParsedOriginManifest, + origin_epoch_handoff: bool, + ) -> Result { + if origin_epoch_handoff && self.should_start_new_origin_epoch_for_handoff(manifest) { + let next_origin_seq = manifest + .segments + .first() + .map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); + self.start_new_origin_epoch(next_origin_seq, OriginEpochTransitionReason::Handoff)?; + return Ok(true); + } + + if self.should_start_new_origin_epoch_for_rollover(manifest) { + let next_origin_seq = manifest + .segments + .first() + .map_or(manifest.origin_manifest_sequence, |segment| segment.origin_seq); + self.start_new_origin_epoch(next_origin_seq, OriginEpochTransitionReason::Rollover)?; + return Ok(true); + } + + Ok(false) + } + + fn apply_forward_jump(&mut self, manifest: &ParsedOriginManifest) -> Result<(), TimelineMapError> { + let Some(highwater) = self.origin_seq_highwater else { + return Ok(()); + }; + let next_origin_seq = highwater.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?; + if manifest.origin_manifest_sequence <= next_origin_seq { + return Ok(()); + } + + let missing_origin_segments = manifest.origin_manifest_sequence - next_origin_seq; + info!( + "HLS forward jump accepted: proxy_session_id={} origin_sequence={} missing_segments={missing_origin_segments}", + safe_proxy_session_id(&self.proxy_session_id), + manifest.origin_manifest_sequence + ); + Ok(()) + } + + fn should_start_new_origin_epoch_for_handoff(&self, manifest: &ParsedOriginManifest) -> bool { + if self.segments.is_empty() { + return false; + } + let Some(highwater) = self.origin_seq_highwater else { + return false; + }; + manifest.segments.iter().any(|segment| segment.origin_seq >= highwater) + } + + fn should_start_new_origin_epoch_for_rollover(&self, manifest: &ParsedOriginManifest) -> bool { + if self.segments.is_empty() { + return false; + } + let Some(highwater) = self.origin_seq_highwater else { + return false; + }; + manifest.segments.last().is_some_and(|segment| segment.origin_seq < highwater) + } + + fn start_new_origin_epoch( + &mut self, + next_origin_seq: u64, + reason: OriginEpochTransitionReason, + ) -> Result<(), TimelineMapError> { + let previous_highwater = self.origin_seq_highwater; + self.origin_epoch = self.origin_epoch.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?; + self.origin_seq_highwater = None; + if let (OriginEpochTransitionReason::Rollover, Some(highwater)) = (reason, previous_highwater) { + info!( + "HLS media sequence rollover detected: proxy_session_id={} previous_highwater={highwater} next_origin_seq={} origin_epoch={}", + safe_proxy_session_id(&self.proxy_session_id), + next_origin_seq, + self.origin_epoch + ); + } + Ok(()) + } + + fn map_origin_segment( + &mut self, + parsed: &ParsedOriginSegment, + manifest: &ParsedOriginManifest, + handoff_discontinuity_before: bool, + ) -> Result { + let current_epoch_key = OriginSegmentKey { origin_epoch: self.origin_epoch, origin_seq: parsed.origin_seq }; + if let Some(proxy_seq) = self.origin_to_proxy.get(¤t_epoch_key).copied() { + if let Some(entry) = self.segments.get_mut(&proxy_seq) { + // Refresh only the concrete fetch reference. Segment identity stays origin_epoch + origin_seq. + entry.origin_fetch_ref = Some(OriginSegmentFetchRef { + resolved_origin_url: parsed.resolved_origin_url.clone(), + byte_range: parsed.origin_byte_range, + valid_until_ms: None, + }); + } + return Ok(proxy_seq); + } + + let mut rollover_discontinuity = false; + match self.origin_seq_highwater { + Some(highwater) if parsed.origin_seq < highwater => { + self.origin_epoch = self.origin_epoch.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?; + self.origin_seq_highwater = Some(parsed.origin_seq); + rollover_discontinuity = true; + info!( + "HLS media sequence rollover detected: proxy_session_id={} previous_highwater={highwater} next_origin_seq={} origin_epoch={}", + safe_proxy_session_id(&self.proxy_session_id), + parsed.origin_seq, + self.origin_epoch + ); + } + Some(highwater) => self.origin_seq_highwater = Some(highwater.max(parsed.origin_seq)), + None => self.origin_seq_highwater = Some(parsed.origin_seq), + } + + let origin_key = OriginSegmentKey { origin_epoch: self.origin_epoch, origin_seq: parsed.origin_seq }; + + let proxy_seq = self.proxy_next_seq.unwrap_or_default(); + self.proxy_next_seq = Some(proxy_seq.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?); + + let map_ref = parsed + .map_ref + .map(|map_id| { + let parsed_map = manifest.maps.get(map_id).ok_or(TimelineMapError::UnsupportedMapExtension)?; + self.map_origin_map(parsed_map) + }) + .transpose()?; + let proxy_file_ext = proxy_extension_from_url(&parsed.resolved_origin_url, SEGMENT_EXTENSIONS) + .ok_or(TimelineMapError::UnsupportedSegmentExtension)?; + let entry = SegmentEntry { + origin_key, + proxy_seq, + duration_ms: parsed.duration_ms, + content_type: default_content_type_for_segment_ext(&proxy_file_ext).to_string(), + cache_key: SegmentCacheKey::new(self.proxy_session_id.clone(), proxy_seq, &proxy_file_ext), + proxy_file_ext, + discontinuity_before: parsed.discontinuity_before || rollover_discontinuity || handoff_discontinuity_before, + program_date_time: parsed.program_date_time.clone(), + daterange_tags_before: parsed.daterange_tags_before.clone(), + origin_byte_range: parsed.origin_byte_range, + map_ref, + // Preserve the concrete fetch target resolved by the parser; do not derive provider/session identity from it. + origin_fetch_ref: Some(OriginSegmentFetchRef { + resolved_origin_url: parsed.resolved_origin_url.clone(), + byte_range: parsed.origin_byte_range, + valid_until_ms: None, + }), + status: SegmentCacheStatus::Discovered, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }; + + self.origin_to_proxy.insert(origin_key, proxy_seq); + self.segments.insert(proxy_seq, entry); + Ok(proxy_seq) + } + + fn map_origin_map(&mut self, parsed_map: &ParsedOriginMap) -> Result { + let origin_map_key = OriginMapKey { + origin_epoch: self.origin_epoch, + resolved_origin_uri: parsed_map.resolved_origin_uri.clone(), + byte_range: parsed_map.byte_range, + }; + if let Some(proxy_map_id) = self.origin_map_to_proxy.get(&origin_map_key).copied() { + return Ok(proxy_map_id); + } + + let proxy_map_id = ProxyMapId(self.next_proxy_map_id); + self.next_proxy_map_id = self.next_proxy_map_id.checked_add(1).ok_or(TimelineMapError::ProxyMapIdOverflow)?; + let proxy_file_ext = proxy_extension_from_url(&origin_map_key.resolved_origin_uri, MAP_EXTENSIONS) + .ok_or(TimelineMapError::UnsupportedMapExtension)?; + self.maps.insert( + proxy_map_id, + MapEntry::new(&self.proxy_session_id, proxy_map_id, origin_map_key.clone(), proxy_file_ext), + ); + self.origin_map_to_proxy.insert(origin_map_key, proxy_map_id); + Ok(proxy_map_id) + } +} + +fn proxy_extension_from_url(url: &str, allowed_extensions: &[&str]) -> Option { + let parsed = Url::parse(url).ok()?; + let file_name = parsed.path_segments()?.next_back()?; + let extension = file_name.rsplit_once('.')?.1.to_ascii_lowercase(); + allowed_extensions.contains(&extension.as_str()).then_some(extension) +} + +#[cfg(test)] +mod tests { + use super::{SegmentCacheStatus, TimelineMapError}; + use crate::{ + api::model::{ + HlsSession, HlsSessionKey, MapCacheStatus, OriginSegmentKey, ProxyMapId, SegmentFetchPriority, + }, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, + }; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn session() -> HlsSession { HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0) } + + fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + #[test] + fn parsed_target_duration_is_stored_for_account_overlap_timing() { + let mut session = session(); + let manifest = + normal_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:12.0,\n10.ts\n"); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + assert_eq!(session.target_duration, Some(12)); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); + assert_eq!(session.account_overlap_timing().hard_active_window_ms, 12_000); + assert_eq!(session.account_overlap_timing().soft_active_window_ms, 24_000); + } + + #[test] + fn origin_rollover_maps_to_monotone_proxy_sequence() { + let mut session = session(); + let first = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:322\n#EXTINF:4.0,\n322.ts\n#EXTINF:4.0,\n323.ts\n#EXTINF:4.0,\n324.ts\n", + ); + let second = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.apply_origin_manifest(&second).expect("second manifest should map"); + + assert_eq!(session.segments.keys().copied().collect::>(), vec![0, 1, 2, 3, 4, 5]); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 0 }), Some(&3)); + assert!(session.segments.get(&3).expect("rollover segment").discontinuity_before); + } + + #[test] + fn origin_rollover_remaps_low_sequences_even_when_previous_epoch_contains_them() { + let mut session = session(); + let old_low = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ); + let high = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:190\n#EXTINF:4.0,\n190.ts\n#EXTINF:4.0,\n191.ts\n#EXTINF:4.0,\n192.ts\n", + ); + let rollover = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ); + + session.apply_origin_manifest(&old_low).expect("old low manifest should map"); + session.apply_origin_manifest(&high).expect("high manifest should map"); + session.apply_origin_manifest(&rollover).expect("rollover manifest should map"); + + assert_eq!(session.origin_epoch, 1); + assert_eq!(session.origin_seq_highwater, Some(2)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 0, origin_seq: 0 }), Some(&0)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 0, origin_seq: 190 }), Some(&3)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 0 }), Some(&6)); + assert_eq!(session.segments.keys().copied().collect::>(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); + assert!(session.segments.get(&6).expect("new epoch segment").discontinuity_before); + } + + #[test] + fn repeated_manifest_after_rollover_reuses_new_epoch_mapping() { + let mut session = session(); + let old_low = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ); + let high = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:190\n#EXTINF:4.0,\n190.ts\n#EXTINF:4.0,\n191.ts\n#EXTINF:4.0,\n192.ts\n", + ); + let rollover = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ); + + session.apply_origin_manifest(&old_low).expect("old low manifest should map"); + session.apply_origin_manifest(&high).expect("high manifest should map"); + session.apply_origin_manifest(&rollover).expect("rollover manifest should map"); + session.apply_origin_manifest(&rollover).expect("same rollover manifest should reuse new epoch"); + + assert_eq!(session.origin_epoch, 1); + assert_eq!(session.origin_seq_highwater, Some(2)); + assert_eq!(session.proxy_next_seq, Some(9)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 0 }), Some(&6)); + assert_eq!(session.segments.len(), 9); + } + + #[test] + fn known_origin_key_is_not_remapped() { + let mut session = session(); + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\n10.ts\n"); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.apply_origin_manifest(&manifest).expect("same manifest should be ignored"); + + assert_eq!(session.segments.len(), 1); + assert_eq!(session.proxy_next_seq, Some(1)); + } + + #[test] + fn overlapping_old_origin_sequences_do_not_trigger_rollover() { + let mut session = session(); + let first = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n", + ); + let overlap = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n#EXTINF:4.0,\n103.ts\n", + ); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.apply_origin_manifest(&overlap).expect("overlapping manifest should map"); + + assert_eq!(session.origin_epoch, 0); + assert_eq!(session.origin_seq_highwater, Some(103)); + assert_eq!(session.segments.keys().copied().collect::>(), vec![0, 1, 2, 3]); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(1)); + assert_eq!(session.publishable_origin_tail_proxy_seq, Some(3)); + } + + #[test] + fn forward_jump_preserves_compact_proxy_sequence() { + let mut session = session(); + let first = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n"); + let jump = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:107\n#EXTINF:4.0,\n107.ts\n#EXTINF:4.0,\n108.ts\n#EXTINF:4.0,\n109.ts\n", + ); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.apply_origin_manifest(&jump).expect("jump manifest should map"); + + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 0, origin_seq: 107 }), Some(&1)); + assert_eq!(session.proxy_next_seq, Some(4)); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(1)); + assert_eq!(session.publishable_origin_tail_proxy_seq, Some(3)); + } + + #[test] + fn host_handoff_remaps_same_origin_sequence_to_new_proxy_sequence() { + let mut session = session(); + let first = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n"); + let second = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n"); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.mark_pending_origin_epoch_handoff_discontinuity(0); + session.apply_origin_manifest(&second).expect("second manifest should map"); + + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 0, origin_seq: 100 }), Some(&0)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 100 }), Some(&1)); + assert!(session.segments.get(&1).expect("host handoff segment").discontinuity_before); + assert_eq!(session.proxy_next_seq, Some(2)); + } + + #[test] + fn host_handoff_starts_new_epoch_when_range_overlaps_highwater() { + let mut session = session(); + let first = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n", + ); + let second = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:99\n#EXTINF:4.0,\n99.ts\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n", + ); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.mark_pending_origin_epoch_handoff_discontinuity(0); + session.apply_origin_manifest(&second).expect("handoff manifest should map"); + + assert_eq!(session.origin_epoch, 1); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 99 }), Some(&2)); + assert_eq!(session.origin_to_proxy.get(&OriginSegmentKey { origin_epoch: 1, origin_seq: 101 }), Some(&4)); + assert!(session.segments.get(&2).expect("handoff head").discontinuity_before); + } + + #[test] + fn mapping_error_does_not_commit_partial_session_state() { + let mut session = session(); + let first = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\n10.ts\n"); + let invalid = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:11\n#EXTINF:4.0,\n11.ts\n#EXTINF:4.0,\n12.webm\n"); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + let previous_proxy_next_seq = session.proxy_next_seq; + let previous_highwater = session.origin_seq_highwater; + let previous_head = session.publishable_origin_head_proxy_seq; + let previous_tail = session.publishable_origin_tail_proxy_seq; + let previous_segments = session.segments.clone(); + let previous_origin_to_proxy = session.origin_to_proxy.clone(); + + assert_eq!(session.apply_origin_manifest(&invalid), Err(TimelineMapError::UnsupportedSegmentExtension)); + + assert_eq!(session.proxy_next_seq, previous_proxy_next_seq); + assert_eq!(session.origin_seq_highwater, previous_highwater); + assert_eq!(session.publishable_origin_head_proxy_seq, previous_head); + assert_eq!(session.publishable_origin_tail_proxy_seq, previous_tail); + assert_eq!(session.segments, previous_segments); + assert_eq!(session.origin_to_proxy, previous_origin_to_proxy); + } + + #[test] + fn unsupported_segment_extension_fails_mapping() { + let mut session = session(); + let manifest = normal_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.webm\n"); + + assert_eq!(session.apply_origin_manifest(&manifest), Err(TimelineMapError::UnsupportedSegmentExtension)); + } + + #[test] + fn mapped_maps_start_as_discovered_placeholders() { + let mut session = session(); + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nseg.m4s\n"); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + let map = session.maps.values().next().expect("map placeholder"); + assert_eq!(map.status, MapCacheStatus::Discovered); + assert_eq!(map.proxy_map_id, ProxyMapId(0)); + assert_eq!(map.proxy_file_ext, "mp4"); + assert_eq!(map.origin_key.origin_epoch, 0); + assert_eq!(map.origin_key.byte_range, None); + assert_eq!(map.cache_key.proxy_map_id(), ProxyMapId(0)); + let fetch_ref = map.origin_fetch_ref.as_ref().expect("map fetch ref"); + assert_eq!(fetch_ref.byte_range, None); + assert!(format!("{fetch_ref:?}").contains("")); + assert!(!format!("{fetch_ref:?}").contains("init.mp4")); + } + + #[test] + fn map_fetch_ref_preserves_final_manifest_host_for_relative_map_uri() { + let mut session = session(); + let manifest = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nseg.m4s\n", + "https://cdn.example.net/live/redirected/playlist.m3u8", + ); + let crate::processing::parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = manifest + else { + panic!("manifest should parse as normal timeline"); + }; + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + let map = session.maps.get(&ProxyMapId(0)).expect("map placeholder"); + assert_eq!(map.origin_key.resolved_origin_uri, "https://cdn.example.net/live/redirected/init.mp4"); + assert_eq!( + map.origin_fetch_ref.as_ref().expect("map fetch ref").resolved_origin_url, + "https://cdn.example.net/live/redirected/init.mp4" + ); + } + + #[test] + fn manifest_mapping_sets_origin_fetch_ref() { + let mut session = session(); + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\nseg.ts\n"); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + let segment = session.segments.get(&0).expect("segment should be mapped"); + let fetch_ref = segment.origin_fetch_ref.as_ref().expect("fetch ref should be set"); + assert!(format!("{fetch_ref:?}").contains("")); + assert!(!format!("{fetch_ref:?}").contains("seg.ts")); + } + + #[test] + fn segment_fetch_ref_preserves_final_manifest_host_for_relative_segment_uri() { + let mut session = session(); + let manifest = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nmedia/seg001.ts\n", + "https://cdn.example.net/live/redirected/playlist.m3u8", + ); + let crate::processing::parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = manifest + else { + panic!("manifest should parse as normal timeline"); + }; + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + let segment = session.segments.get(&0).expect("segment should be mapped"); + assert_eq!(segment.origin_key, OriginSegmentKey { origin_epoch: 0, origin_seq: 10 }); + assert_eq!( + segment.origin_fetch_ref.as_ref().expect("fetch ref").resolved_origin_url, + "https://cdn.example.net/live/redirected/media/seg001.ts" + ); + } + + #[test] + fn cold_start_prefetch_prioritizes_visible_window_then_known_tail() { + let mut session = session(); + session.initial_prefetch_gap_segments = 3; + session.configure_segment_prefetch_queue(6); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n100.ts\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n#EXTINF:4.0,\n103.ts\n#EXTINF:4.0,\n104.ts\n#EXTINF:4.0,\n105.ts\n", + ); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.queue_manifest_prefetch_candidates(10); + + assert!(matches!( + session.segments.get(&0).expect("0").status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. } + )); + assert!(matches!( + session.segments.get(&2).expect("2").status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. } + )); + assert!(matches!( + session.segments.get(&3).expect("3").status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, .. } + )); + assert_eq!(session.segment_prefetch_queue.prefetch_len(), 3); + } + + #[test] + fn same_map_uri_with_different_byterange_uses_distinct_proxy_map_ids() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@0\"\n#EXTINF:4.0,\n1.m4s\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"100@100\"\n#EXTINF:4.0,\n2.m4s\n", + ); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + assert_eq!(session.maps.len(), 2); + assert_eq!(session.segments.get(&0).expect("first segment").map_ref, Some(ProxyMapId(0))); + assert_eq!(session.segments.get(&1).expect("second segment").map_ref, Some(ProxyMapId(1))); + } + + #[test] + fn same_map_uri_in_same_epoch_reuses_proxy_map_id() { + let mut session = session(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n2.m4s\n", + ); + + session.apply_origin_manifest(&manifest).expect("manifest should map"); + + assert_eq!(session.maps.len(), 1); + assert_eq!(session.segments.get(&0).expect("first segment").map_ref, Some(ProxyMapId(0))); + assert_eq!(session.segments.get(&1).expect("second segment").map_ref, Some(ProxyMapId(0))); + } + + #[test] + fn same_map_uri_after_rollover_uses_distinct_proxy_map_id() { + let mut session = session(); + let first = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:322\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n322.m4s\n"); + let second = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n0.m4s\n"); + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.apply_origin_manifest(&second).expect("second manifest should map"); + + assert_eq!(session.maps.len(), 2); + assert_eq!(session.segments.get(&0).expect("first epoch segment").map_ref, Some(ProxyMapId(0))); + assert_eq!(session.segments.get(&1).expect("second epoch segment").map_ref, Some(ProxyMapId(1))); + assert_eq!(session.maps.get(&ProxyMapId(1)).expect("second map").origin_key.origin_epoch, 1); + } + + #[test] + fn same_relative_map_uri_on_different_final_hosts_uses_distinct_proxy_map_ids() { + let mut session = session(); + let first = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n0.m4s\n", + "https://cdn-a.example.net/live/playlist.m3u8", + ); + let second = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", + "https://cdn-b.example.net/live/playlist.m3u8", + ); + let crate::processing::parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(first) = first else { + panic!("first manifest should parse as normal timeline"); + }; + let crate::processing::parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(second) = second else { + panic!("second manifest should parse as normal timeline"); + }; + + session.apply_origin_manifest(&first).expect("first manifest should map"); + session.apply_origin_manifest(&second).expect("second manifest should map"); + + assert_eq!(session.maps.len(), 2); + assert_eq!( + session.maps.get(&ProxyMapId(0)).expect("first map").origin_key.resolved_origin_uri, + "https://cdn-a.example.net/live/init.mp4" + ); + assert_eq!( + session.maps.get(&ProxyMapId(1)).expect("second map").origin_key.resolved_origin_uri, + "https://cdn-b.example.net/live/init.mp4" + ); + assert_eq!(session.segments.get(&0).expect("first segment").map_ref, Some(ProxyMapId(0))); + assert_eq!(session.segments.get(&1).expect("second segment").map_ref, Some(ProxyMapId(1))); + } +} diff --git a/backend/src/api/model/hls_cache/transient.rs b/backend/src/api/model/hls_cache/transient.rs new file mode 100644 index 000000000..6e0cde3e5 --- /dev/null +++ b/backend/src/api/model/hls_cache/transient.rs @@ -0,0 +1,659 @@ +use super::{CacheAccessState, ProxySessionId, TransientObjectCacheKey}; +use axum::http::StatusCode; +use base64::{engine::general_purpose, Engine as _}; +use std::{ + collections::{HashMap, HashSet}, + fmt, + sync::Arc, +}; +use tokio::sync::Notify; + +const TRANSIENT_RESOURCE_ID_LEN: usize = 16; +const DEFAULT_TRANSIENT_RESOURCE_TTL_MS: u64 = 300_000; +const TRANSIENT_RESOURCE_ID_KEY_CONTEXT: &str = "tuliprox:hls-cache:transient-resource-id-key:v1"; + +/// Opaque ID for a transient passthrough resource. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct TransientResourceId(pub String); + +/// Builds a deterministic opaque transient resource ID from the concrete origin fetch URI. +/// +/// `resolved_origin_uri` must be the concrete resource URI after manifest-relative URL resolution against the final +/// manifest URL. In provider-url-failover/redirect flows this may intentionally include the selected mirror or final +/// CDN/origin host. Do not make this ID input host-neutral unless `TransientResourceRef` keeps a separate concrete fetch +/// URI and tests prove relative segment/MAP/key downloads still use that concrete URI. +/// +pub fn build_transient_resource_id( + resolved_origin_uri: &str, + reverse_proxy_rewrite_secret: &[u8], +) -> TransientResourceId { + let key = blake3::derive_key(TRANSIENT_RESOURCE_ID_KEY_CONTEXT, reverse_proxy_rewrite_secret); + let digest = blake3::keyed_hash(&key, resolved_origin_uri.as_bytes()); + let token = general_purpose::URL_SAFE_NO_PAD.encode(digest.as_bytes()); + TransientResourceId(token.chars().take(TRANSIENT_RESOURCE_ID_LEN).collect()) +} + +/// Transient origin resource category used for direct passthrough streaming. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum TransientResourceKind { + Segment, + Key, + Map, + Other, +} + +/// Per-session mapping from an opaque transient ID to one resolved origin resource. +#[derive(Clone, Eq, PartialEq)] +pub struct TransientResourceRef { + pub id: TransientResourceId, + pub kind: TransientResourceKind, + /// Concrete origin fetch URI for this transient resource. + /// + /// This is request-local fetch metadata, not HLS session identity. It must remain the final concrete URI produced + /// after resolving relative segment/MAP/key references against the final manifest URL. + pub resolved_origin_uri: String, + pub content_type_hint: Option, + pub file_ext_hint: Option, + pub created_at_ms: u64, + pub expires_at_ms: u64, + pub access: Arc, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TransientObjectCacheStatus { + Fetching { started_at_ms: u64 }, + Ready { content_length: u64, ready_at_ms: u64 }, + FailedRetryable { failed_at_ms: u64, retry_after_ms: u64 }, + FailedPermanent { failed_at_ms: u64, status: Option }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum TransientObjectUnavailableState { + Missing, + Fetching, + FailedRetryable { retry_after_ms: u64 }, + FailedPermanent, +} + +#[derive(Clone, Eq, PartialEq)] +pub struct TransientObjectCacheEntry { + pub key: TransientObjectCacheKey, + pub status: TransientObjectCacheStatus, + pub content_type: String, + pub created_at_ms: u64, + pub last_accessed_at_ms: u64, + pub expires_at_ms: u64, + pub access: Arc, +} + +#[derive(Clone)] +pub enum TransientObjectFetchDecision { + Ready, + Fetch(TransientObjectCacheKey), + Wait(Arc), +} + +impl TransientObjectCacheEntry { + fn new_fetching(key: TransientObjectCacheKey, now_ms: u64, expires_at_ms: u64, content_type: String) -> Self { + Self { + key, + status: TransientObjectCacheStatus::Fetching { started_at_ms: now_ms }, + content_type, + created_at_ms: now_ms, + last_accessed_at_ms: now_ms, + expires_at_ms, + access: Arc::new(CacheAccessState::new()), + } + } + + pub fn is_ready_at(&self, now_ms: u64) -> bool { + matches!(self.status, TransientObjectCacheStatus::Ready { .. }) && self.expires_at_ms >= now_ms + } + + pub fn ready_content_length(&self) -> Option { + match self.status { + TransientObjectCacheStatus::Ready { content_length, .. } => Some(content_length), + TransientObjectCacheStatus::Fetching { .. } + | TransientObjectCacheStatus::FailedRetryable { .. } + | TransientObjectCacheStatus::FailedPermanent { .. } => None, + } + } +} + +impl fmt::Debug for TransientObjectCacheEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientObjectCacheEntry") + .field("key", &self.key) + .field("status", &self.status) + .field("content_type", &self.content_type) + .field("created_at_ms", &self.created_at_ms) + .field("last_accessed_at_ms", &self.last_accessed_at_ms) + .field("expires_at_ms", &self.expires_at_ms) + .field("active_readers", &self.access.active_readers()) + .finish() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct TransientObjectRemoval { + pub key: TransientObjectCacheKey, + pub content_length: u64, +} + +impl TransientResourceRef { + pub fn new( + kind: TransientResourceKind, + resolved_origin_uri: impl Into, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + file_ext_hint: Option, + ) -> Self { + let resolved_origin_uri = resolved_origin_uri.into(); + let id = build_transient_resource_id(&resolved_origin_uri, reverse_proxy_rewrite_secret); + Self { + id, + kind, + resolved_origin_uri, + content_type_hint: file_ext_hint + .as_deref() + .and_then(default_content_type_for_transient_ext) + .map(str::to_string), + file_ext_hint, + created_at_ms: now_ms, + expires_at_ms: now_ms.saturating_add(ttl_ms), + access: Arc::new(CacheAccessState::new()), + } + } + + pub fn is_valid_at(&self, now_ms: u64) -> bool { now_ms <= self.expires_at_ms } + + fn refresh_from(&mut self, next: TransientResourceRef) { + self.kind = next.kind; + self.resolved_origin_uri = next.resolved_origin_uri; + self.content_type_hint = next.content_type_hint; + self.file_ext_hint = next.file_ext_hint; + self.expires_at_ms = next.expires_at_ms; + } + + pub fn active_readers(&self) -> u32 { self.access.active_readers() } +} + +impl fmt::Debug for TransientResourceRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientResourceRef") + .field("id", &self.id) + .field("kind", &self.kind) + .field("resolved_origin_uri", &"") + .field("content_type_hint", &self.content_type_hint) + .field("file_ext_hint", &self.file_ext_hint) + .field("created_at_ms", &self.created_at_ms) + .field("expires_at_ms", &self.expires_at_ms) + .field("active_readers", &self.access.active_readers()) + .finish() + } +} + +/// Per-session transient passthrough manifest and resource mappings. +#[derive(Clone)] +pub struct TransientPassthroughState { + pub resources: HashMap, + pub object_cache: HashMap, + object_fetch_notifiers: HashMap>, + pub last_manifest_body: Option, + pub last_manifest_rendered_at_ms: Option, + pub last_manifest_playlist_duration_ms: Option, + pub last_manifest_valid_until_ms: Option, + pub resource_ttl_ms: u64, +} + +impl TransientPassthroughState { + pub fn new(resource_ttl_ms: u64) -> Self { + Self { + resources: HashMap::new(), + object_cache: HashMap::new(), + object_fetch_notifiers: HashMap::new(), + last_manifest_body: None, + last_manifest_rendered_at_ms: None, + last_manifest_playlist_duration_ms: None, + last_manifest_valid_until_ms: None, + resource_ttl_ms, + } + } + + pub fn set_resource_ttl_ms(&mut self, resource_ttl_ms: u64) { self.resource_ttl_ms = resource_ttl_ms; } + + pub fn replace_manifest(&mut self, body: String, rendered_at_ms: u64) { + self.last_manifest_body = Some(body); + self.last_manifest_rendered_at_ms = Some(rendered_at_ms); + self.last_manifest_playlist_duration_ms = None; + self.last_manifest_valid_until_ms = None; + } + + pub fn replace_manifest_with_validity( + &mut self, + body: String, + rendered_at_ms: u64, + playlist_duration_ms: u64, + ) { + self.last_manifest_body = Some(body); + self.last_manifest_rendered_at_ms = Some(rendered_at_ms); + self.last_manifest_playlist_duration_ms = Some(playlist_duration_ms); + self.last_manifest_valid_until_ms = Some(rendered_at_ms.saturating_add(playlist_duration_ms)); + } + + pub fn upsert_resources(&mut self, resources: I) + where + I: IntoIterator, + { + for resource in resources { + let resource_id = resource.id.clone(); + let expires_at_ms = resource.expires_at_ms; + match self.resources.get_mut(&resource.id) { + Some(existing) => existing.refresh_from(resource), + None => { + self.resources.insert(resource.id.clone(), resource); + } + } + self.extend_object_ttl_for_resource(&resource_id, expires_at_ms); + } + } + + fn extend_object_ttl_for_resource(&mut self, resource_id: &TransientResourceId, expires_at_ms: u64) { + for (key, entry) in &mut self.object_cache { + if key.transient_resource_id() == resource_id { + entry.expires_at_ms = entry.expires_at_ms.max(expires_at_ms); + } + } + } + + pub fn get_valid_resource( + &mut self, + resource_id: &TransientResourceId, + now_ms: u64, + ) -> Option { + self.prune_expired(now_ms); + self.resources.get(resource_id).cloned() + } + + pub fn prune_expired(&mut self, now_ms: u64) { + let protected = self.protected_manifest_resource_ids(); + self.prune_expired_except(now_ms, &protected); + } + + pub fn prune_expired_except(&mut self, now_ms: u64, protected: &HashSet) { + self.resources.retain(|id, resource| { + protected.contains(id) || resource.is_valid_at(now_ms) || resource.active_readers() > 0 + }); + } + + pub fn protected_manifest_resource_ids(&self) -> HashSet { + self.last_manifest_body.as_deref().map_or_else(HashSet::new, extract_transient_resource_ids) + } + + pub fn active_resource_readers(&self) -> u32 { + self.resources.values().map(TransientResourceRef::active_readers).sum() + } + + pub fn has_active_resource_readers(&self) -> bool { self.active_resource_readers() > 0 } + + /// Builds the object-cache key for an already registered transient resource. + /// + /// The key is intentionally based on the opaque resource ID. The concrete fetch URI remains in + /// `TransientResourceRef::resolved_origin_uri` and must not be reconstructed from this key. + pub fn transient_object_key( + proxy_session_id: &ProxySessionId, + resource_id: &TransientResourceId, + file_ext: impl Into, + ) -> TransientObjectCacheKey { + TransientObjectCacheKey::new(proxy_session_id.clone(), resource_id.clone(), file_ext) + } + + pub fn ready_object(&mut self, key: &TransientObjectCacheKey, now_ms: u64) -> Option { + let entry = self.object_cache.get_mut(key)?; + if !entry.is_ready_at(now_ms) { + return None; + } + entry.last_accessed_at_ms = now_ms; + entry.access.reader_started(now_ms); + entry.access.reader_finished(); + Some(entry.clone()) + } + + pub fn begin_object_fetch( + &mut self, + proxy_session_id: &ProxySessionId, + resource: &TransientResourceRef, + file_ext: &str, + now_ms: u64, + cache_duration_ms: u64, + ) -> TransientObjectFetchDecision { + let key = Self::transient_object_key(proxy_session_id, &resource.id, file_ext.to_string()); + match self.object_cache.get(&key) { + Some(entry) if entry.is_ready_at(now_ms) => return TransientObjectFetchDecision::Ready, + Some(entry) if matches!(entry.status, TransientObjectCacheStatus::Fetching { .. }) => { + let notifier = + self.object_fetch_notifiers.entry(key).or_insert_with(|| Arc::new(Notify::new())).clone(); + return TransientObjectFetchDecision::Wait(notifier); + } + Some(_) | None => {} + } + let expires_at_ms = now_ms.saturating_add(cache_duration_ms).max(resource.expires_at_ms); + let content_type = resource.content_type_hint.clone().unwrap_or_else(|| "application/octet-stream".to_string()); + self.object_cache.insert( + key.clone(), + TransientObjectCacheEntry::new_fetching(key.clone(), now_ms, expires_at_ms, content_type), + ); + self.object_fetch_notifiers.entry(key.clone()).or_insert_with(|| Arc::new(Notify::new())); + TransientObjectFetchDecision::Fetch(key) + } + + pub fn mark_object_ready( + &mut self, + key: &TransientObjectCacheKey, + content_type: String, + content_length: u64, + now_ms: u64, + expires_at_ms: u64, + ) { + let notify_waiters = self.object_fetch_notifiers.remove(key); + match self.object_cache.get_mut(key) { + Some(entry) => { + entry.status = TransientObjectCacheStatus::Ready { content_length, ready_at_ms: now_ms }; + entry.content_type = content_type; + entry.last_accessed_at_ms = now_ms; + entry.expires_at_ms = expires_at_ms; + } + None => { + self.object_cache.insert( + key.clone(), + TransientObjectCacheEntry { + key: key.clone(), + status: TransientObjectCacheStatus::Ready { content_length, ready_at_ms: now_ms }, + content_type, + created_at_ms: now_ms, + last_accessed_at_ms: now_ms, + expires_at_ms, + access: Arc::new(CacheAccessState::new()), + }, + ); + } + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + } + + pub fn mark_object_failed_retryable(&mut self, key: &TransientObjectCacheKey, now_ms: u64, retry_after_ms: u64) { + let notify_waiters = self.object_fetch_notifiers.remove(key); + if let Some(entry) = self.object_cache.get_mut(key) { + entry.status = TransientObjectCacheStatus::FailedRetryable { failed_at_ms: now_ms, retry_after_ms }; + entry.last_accessed_at_ms = now_ms; + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + } + + pub fn mark_object_failed_permanent( + &mut self, + key: &TransientObjectCacheKey, + now_ms: u64, + status: Option, + ) { + let notify_waiters = self.object_fetch_notifiers.remove(key); + if let Some(entry) = self.object_cache.get_mut(key) { + entry.status = TransientObjectCacheStatus::FailedPermanent { failed_at_ms: now_ms, status }; + entry.last_accessed_at_ms = now_ms; + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + } + + pub fn object_status(&self, key: &TransientObjectCacheKey) -> Option { + self.object_cache.get(key).map(|entry| entry.status.clone()) + } + + pub fn object_unavailable_state( + &self, + key: &TransientObjectCacheKey, + now_ms: u64, + ) -> TransientObjectUnavailableState { + let Some(entry) = self.object_cache.get(key) else { + return TransientObjectUnavailableState::Missing; + }; + if entry.expires_at_ms < now_ms { + return TransientObjectUnavailableState::Missing; + } + match entry.status { + TransientObjectCacheStatus::Fetching { .. } => TransientObjectUnavailableState::Fetching, + TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. } => { + TransientObjectUnavailableState::FailedRetryable { retry_after_ms } + } + TransientObjectCacheStatus::FailedPermanent { .. } => TransientObjectUnavailableState::FailedPermanent, + TransientObjectCacheStatus::Ready { .. } => TransientObjectUnavailableState::Missing, + } + } + + pub fn ready_object_cache_size(&self) -> u64 { + self.object_cache.values().filter_map(TransientObjectCacheEntry::ready_content_length).sum() + } + + pub fn prune_expired_objects(&mut self, now_ms: u64) -> Vec { + let keys = self + .object_cache + .iter() + .filter_map(|(key, entry)| { + if entry.access.active_readers() == 0 && entry.expires_at_ms < now_ms { + return entry.ready_content_length().map(|content_length| (key.clone(), content_length)); + } + None + }) + .collect::>(); + self.remove_object_keys(keys) + } + + pub fn remove_oldest_ready_object(&mut self) -> Option { + let candidate = self + .object_cache + .iter() + .filter_map(|(key, entry)| { + if entry.access.active_readers() == 0 { + return entry.ready_content_length().map(|content_length| { + (key.clone(), content_length, entry.last_accessed_at_ms, entry.created_at_ms) + }); + } + None + }) + .min_by_key(|(_, _, last_accessed_at_ms, created_at_ms)| (*last_accessed_at_ms, *created_at_ms))?; + self.remove_object_keys(vec![(candidate.0, candidate.1)]).into_iter().next() + } + + fn remove_object_keys(&mut self, keys: Vec<(TransientObjectCacheKey, u64)>) -> Vec { + keys.into_iter() + .filter_map(|(key, content_length)| { + self.object_cache.remove(&key)?; + Some(TransientObjectRemoval { key, content_length }) + }) + .collect() + } +} + +impl Default for TransientPassthroughState { + fn default() -> Self { Self::new(DEFAULT_TRANSIENT_RESOURCE_TTL_MS) } +} + +impl fmt::Debug for TransientPassthroughState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientPassthroughState") + .field("resources_len", &self.resources.len()) + .field("object_cache_len", &self.object_cache.len()) + .field("object_fetch_notifiers_len", &self.object_fetch_notifiers.len()) + .field("last_manifest_body_len", &self.last_manifest_body.as_ref().map(String::len)) + .field("last_manifest_rendered_at_ms", &self.last_manifest_rendered_at_ms) + .field("last_manifest_playlist_duration_ms", &self.last_manifest_playlist_duration_ms) + .field("last_manifest_valid_until_ms", &self.last_manifest_valid_until_ms) + .field("resource_ttl_ms", &self.resource_ttl_ms) + .finish() + } +} + +fn extract_transient_resource_ids(body: &str) -> HashSet { + body.split("/r/") + .skip(1) + .filter_map(|tail| { + let file_name: String = + tail.chars().take_while(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')).collect(); + let (resource_id, extension) = file_name.rsplit_once('.')?; + if resource_id.is_empty() || extension.is_empty() { + return None; + } + Some(TransientResourceId(resource_id.to_string())) + }) + .collect() +} + +/// Empty runtime store for future transient passthrough resources. +#[derive(Default)] +pub struct TransientResourceStore; + +impl TransientResourceStore { + pub fn new() -> Self { Self } +} + +fn default_content_type_for_transient_ext(extension: &str) -> Option<&'static str> { + match extension { + "ts" => Some("video/mp2t"), + "mp4" | "m4s" | "m4v" => Some("video/mp4"), + "key" => Some("application/octet-stream"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::{ + build_transient_resource_id, TransientObjectCacheStatus, TransientPassthroughState, TransientResourceKind, + TransientResourceRef, + }; + use crate::api::model::ProxySessionId; + + #[test] + fn transient_resource_id_is_stable_and_opaque() { + let first = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); + let second = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); + let other = build_transient_resource_id("http://origin.example.com/live/seg.ts", b"secret"); + + assert_eq!(first, second); + assert_ne!(first, other); + assert_eq!(first.0.len(), 16); + assert!(!first.0.contains("origin")); + } + + #[test] + fn transient_state_updates_existing_resource_ttl() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + let updated = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 20, + 200, + Some("ts".to_string()), + ); + state.upsert_resources([updated]); + + assert_eq!(state.resources.len(), 1); + assert_eq!(state.get_valid_resource(&resource_id, 150).expect("resource remains valid").expires_at_ms, 220); + assert!(state.get_valid_resource(&resource_id, 221).is_none()); + } + + #[test] + fn transient_resource_render_extends_existing_object_ttl() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + let key = TransientPassthroughState::transient_object_key( + &ProxySessionId("proxy-session".to_string()), + &resource_id, + "ts", + ); + state.upsert_resources([resource.clone()]); + assert!(matches!( + state.begin_object_fetch(&ProxySessionId("proxy-session".to_string()), &resource, "ts", 20, 50), + super::TransientObjectFetchDecision::Fetch(_) + )); + assert_eq!(state.object_cache.get(&key).expect("object").expires_at_ms, 110); + + let updated = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 100, + 300, + Some("ts".to_string()), + ); + state.upsert_resources([updated]); + + let object = state.object_cache.get(&key).expect("object remains"); + assert!(matches!(object.status, TransientObjectCacheStatus::Fetching { .. })); + assert_eq!(object.expires_at_ms, 400); + } + + #[test] + fn transient_media_resource_extensions_use_video_mp4_content_type() { + for extension in ["mp4", "m4s", "m4v"] { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + format!("http://origin.example.com/live/seg.{extension}"), + b"secret", + 10, + 100, + Some(extension.to_string()), + ); + + assert_eq!(resource.content_type_hint.as_deref(), Some("video/mp4")); + } + } + + #[test] + fn transient_prune_keeps_resources_referenced_by_last_manifest() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + state.replace_manifest( + format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n", resource_id.0), + 0, + ); + + state.prune_expired(20); + + assert!(state.resources.contains_key(&resource_id)); + } +} diff --git a/backend/src/api/model/hls_cache/transient_fetcher.rs b/backend/src/api/model/hls_cache/transient_fetcher.rs new file mode 100644 index 000000000..39413bcbe --- /dev/null +++ b/backend/src/api/model/hls_cache/transient_fetcher.rs @@ -0,0 +1,545 @@ +use super::{ + build_hls_origin_resource_headers_with_client_range, + finish_hls_origin_account_io, hls_client_body_send_deadline, hls_object_body_deadline, + refresh_hls_client_body_send_deadline, + log_hls_resource_timeout, run_hls_origin_resource_retry_loop_with_attempt_prepare, CacheAccessState, + HlsAccessLeaseId, HlsMediaActivityMarker, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, + HlsOriginIoContext, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsRepairRenderedObjectId, HlsResourceFetchAttempt, HlsResourceFetchKind, HlsResourceFetchLogContext, + HlsResourceFetchSource, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, HlsSessionHandle, ProxySessionId, SegmentFetchPolicy, TransientObjectCacheKey, + TransientObjectFetchDecision, TransientPassthroughState, TransientResourceFile, TransientResourceKind, + TransientResourceRef, +}; +use crate::api::api_utils::try_unwrap_body; +use axum::{body::Body, http::{header, HeaderMap, HeaderValue, StatusCode}, response::IntoResponse}; +use futures::{future::BoxFuture, FutureExt, StreamExt, TryStreamExt}; +use std::{io, sync::Arc}; +use tokio::{sync::Notify, time::sleep}; +use tokio_util::io::StreamReader; + +pub enum HlsTransientObjectFetchFailure { + Retryable, + Permanent { status: Option }, +} + +pub fn hls_transient_object_fetch_failure(error: &HlsOriginResourceFetchError) -> HlsTransientObjectFetchFailure { + match error { + HlsOriginResourceFetchError::PermanentStatus(status) + | HlsOriginResourceFetchError::NonRetryableStatus(status) => { + HlsTransientObjectFetchFailure::Permanent { status: Some(*status) } + } + HlsOriginResourceFetchError::InvalidOriginUrl + | HlsOriginResourceFetchError::InvalidByteRange + | HlsOriginResourceFetchError::UnexpectedByteRangeStatus => { + HlsTransientObjectFetchFailure::Permanent { status: None } + } + HlsOriginResourceFetchError::RetryableStatus(_) + | HlsOriginResourceFetchError::Transport(_) + | HlsOriginResourceFetchError::Redirect + | HlsOriginResourceFetchError::Timeout + | HlsOriginResourceFetchError::CacheCommit(_) => HlsTransientObjectFetchFailure::Retryable, + HlsOriginResourceFetchError::ProviderUnavailable(kind) if kind.is_retryable_resource_failure() => { + HlsTransientObjectFetchFailure::Retryable + } + HlsOriginResourceFetchError::ProviderUnavailable(_) => HlsTransientObjectFetchFailure::Permanent { status: None }, + } +} + +pub fn hls_transient_resource_fetch_kind(resource_kind: TransientResourceKind) -> HlsResourceFetchKind { + match resource_kind { + TransientResourceKind::Key => HlsResourceFetchKind::Key, + TransientResourceKind::Map => HlsResourceFetchKind::Map, + TransientResourceKind::Segment => HlsResourceFetchKind::Segment, + TransientResourceKind::Other => HlsResourceFetchKind::Other, + } +} + +fn build_hls_transient_resource_fetch_target( + resolved_origin_uri: &str, + origin_headers: &HeaderMap, + origin_provider_session_headers: &HeaderMap, + range_header: Option, + resource_id: &str, + resource_kind: TransientResourceKind, +) -> HlsOriginResourceFetchTarget { + HlsOriginResourceFetchTarget { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: resource_id.to_string(), + origin_url: resolved_origin_uri.to_string(), + headers: build_hls_origin_resource_headers_with_client_range( + origin_headers, + origin_provider_session_headers, + range_header, + ), + byte_range_expectation: HlsOriginByteRangeExpectation::AnySuccess, + } +} + +pub struct HlsTransientOriginFetchRequest { + pub resolved_origin_uri: String, + pub origin_headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub range_header: Option, + pub resource_file: TransientResourceFile, + pub resource_kind: TransientResourceKind, + pub clients: HlsOriginResourceClients, + pub policy: SegmentFetchPolicy, + pub session_log_id: String, +} + +pub async fn fetch_hls_transient_origin_response_with_attempt_prepare( + request: HlsTransientOriginFetchRequest, + prepare_attempt: P, +) -> Result<(reqwest::Response, G), HlsOriginResourceFetchError> +where + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, +{ + let target = build_hls_transient_resource_fetch_target( + &request.resolved_origin_uri, + &request.origin_headers, + &request.origin_provider_session_headers, + request.range_header, + request.resource_file.resource_id.0.as_str(), + request.resource_kind, + ); + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + request.clients, + &request.policy, + &request.session_log_id, + prepare_attempt, + |guard| async move { drop(guard) }.boxed(), + |response, _attempt, guard| async move { Ok((response, guard)) }.boxed(), + ) + .await +} + +pub enum HlsTransientObjectCacheAction { + ServeReady, + FetchAndCache(TransientObjectCacheKey), + WaitForFetch(Arc), + PassthroughNoCache, +} + +pub struct HlsTransientObjectCacheResolution { + pub resource: TransientResourceRef, + pub origin_headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub action: HlsTransientObjectCacheAction, +} + +pub async fn resolve_hls_transient_object_cache_action( + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + resource_file: &TransientResourceFile, + range_header: Option<&HeaderValue>, + now_ms: u64, + cache_duration_ms: u64, +) -> Result { + // `is_gc_marked_for_removal` is `&self` and is the dominant early-exit on a + // busy session (GC sweeps mark sessions on a timer). Resolve it under a read + // lock so we don't pay for the exclusive write-lock acquisition when the + // session is already doomed. + if session.read().await.is_gc_marked_for_removal() { + return Err(StatusCode::NOT_FOUND); + } + let mut session = session.write().await; + let Some(resource) = session.transient.get_valid_resource(&resource_file.resource_id, now_ms) else { + return Err(StatusCode::NOT_FOUND); + }; + if resource.file_ext_hint.as_deref().is_some_and(|extension| extension != resource_file.extension) { + return Err(StatusCode::NOT_FOUND); + } + let action = transient_object_cache_action( + &mut session, + proxy_session_id, + &resource, + resource_file, + range_header, + now_ms, + cache_duration_ms, + ); + Ok(HlsTransientObjectCacheResolution { + resource, + origin_headers: session.origin_request_headers.clone(), + origin_provider_session_headers: session.origin_provider_session_headers.clone(), + action, + }) +} + +fn transient_object_cache_action( + session: &mut super::HlsSession, + proxy_session_id: &ProxySessionId, + resource: &TransientResourceRef, + resource_file: &TransientResourceFile, + range_header: Option<&HeaderValue>, + now_ms: u64, + cache_duration_ms: u64, +) -> HlsTransientObjectCacheAction { + if matches!(resource.kind, TransientResourceKind::Key) { + return HlsTransientObjectCacheAction::PassthroughNoCache; + } + let cache_key = + TransientPassthroughState::transient_object_key(proxy_session_id, &resource.id, resource_file.extension.clone()); + if session.transient.ready_object(&cache_key, now_ms).is_some() { + return HlsTransientObjectCacheAction::ServeReady; + } + if !is_hls_transient_full_object_cacheable_request(range_header) { + return HlsTransientObjectCacheAction::PassthroughNoCache; + } + session + .transient + .begin_object_fetch(proxy_session_id, resource, &resource_file.extension, now_ms, cache_duration_ms) + .into() +} + +impl From for HlsTransientObjectCacheAction { + fn from(decision: TransientObjectFetchDecision) -> Self { + match decision { + TransientObjectFetchDecision::Ready => Self::ServeReady, + TransientObjectFetchDecision::Fetch(cache_key) => Self::FetchAndCache(cache_key), + TransientObjectFetchDecision::Wait(notifier) => Self::WaitForFetch(notifier), + } + } +} + +pub fn is_hls_transient_full_object_cacheable_request(range_header: Option<&HeaderValue>) -> bool { + let Some(range_header) = range_header else { + return true; + }; + range_header.to_str().is_ok_and(|range| range.trim() == "bytes=0-") +} + +pub struct HlsTransientObjectFetchFinalizer { + session: HlsSessionHandle, + cache_key: TransientObjectCacheKey, + completed: bool, + retry_after_ms: u64, +} + +impl HlsTransientObjectFetchFinalizer { + pub fn new(session: HlsSessionHandle, cache_key: TransientObjectCacheKey, retry_after_ms: u64) -> Self { + Self { session, cache_key, completed: false, retry_after_ms } + } + + pub fn complete(&mut self) { self.completed = true; } +} + +impl Drop for HlsTransientObjectFetchFinalizer { + fn drop(&mut self) { + if self.completed { + return; + } + let session = Arc::clone(&self.session); + let cache_key = self.cache_key.clone(); + let retry_after_ms = self.retry_after_ms; + tokio::spawn(async move { + session.write().await.transient.mark_object_failed_retryable( + &cache_key, + current_time_millis(), + retry_after_ms, + ); + }); + } +} + +#[derive(Clone)] +pub struct HlsTransientCacheCommitContext { + pub segment_cache: Arc, + pub segment_repair: Arc, + pub session: HlsSessionHandle, + pub access_lease_id: HlsAccessLeaseId, + pub resource: TransientResourceRef, + pub resource_file: TransientResourceFile, + pub cache_key: TransientObjectCacheKey, + pub range_header: Option, + pub cache_duration_ms: u64, + pub origin_segment_timeout_ms: u64, +} + +pub struct HlsTransientOriginCacheFetchRequest { + pub fetch: HlsTransientOriginFetchRequest, + pub commit: HlsTransientCacheCommitContext, +} + +pub async fn fetch_and_commit_hls_transient_origin_response_with_attempt_prepare( + request: HlsTransientOriginCacheFetchRequest, + prepare_attempt: P, +) -> Result<(), HlsOriginResourceFetchError> +where + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, +{ + let target = build_hls_transient_resource_fetch_target( + &request.fetch.resolved_origin_uri, + &request.fetch.origin_headers, + &request.fetch.origin_provider_session_headers, + request.fetch.range_header, + request.fetch.resource_file.resource_id.0.as_str(), + request.fetch.resource_kind, + ); + let commit = request.commit; + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + request.fetch.clients, + &request.fetch.policy, + &request.fetch.session_log_id, + prepare_attempt, + |guard| async move { drop(guard) }.boxed(), + move |response, _attempt, guard| { + let commit = commit.clone(); + async move { + let result = commit_hls_transient_origin_response_attempt(commit, response).await; + drop(guard); + result + } + .boxed() + }, + ) + .await +} + +async fn commit_hls_transient_origin_response_attempt( + context: HlsTransientCacheCommitContext, + response: reqwest::Response, +) -> Result<(), HlsOriginResourceFetchError> { + let response_headers = response.headers().clone(); + let content_type = response_headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string) + .or_else(|| context.resource.content_type_hint.clone()) + .unwrap_or_else(|| "application/octet-stream".to_string()); + let deadline = hls_object_body_deadline(context.origin_segment_timeout_ms); + let stream_reader = StreamReader::new(response.bytes_stream().map_err(io::Error::other)); + let repair_context = HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Transient, + proxy_session_id: context.session.read().await.proxy_session_id.clone(), + hls_access_lease_id: Some(context.access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Transient { + resource_id: context.resource_file.resource_id.0.clone(), + }, + resource_id: context.resource_file.resource_id.0.clone(), + file_ext: context.resource_file.extension.clone(), + origin_fetch_uri_for_diagnostics: context.resource.resolved_origin_uri.clone(), + media_sequence: None, + discontinuity_sequence: None, + complete_object: is_hls_transient_full_object_cacheable_request(context.range_header.as_ref()), + encrypted: context.resource.kind == TransientResourceKind::Key, + custom_response: false, + }; + let commit = Box::pin(context.segment_repair.commit_origin_response( + &context.segment_cache, + &context.cache_key, + stream_reader, + deadline, + repair_context, + )) + .await; + let ready_at_ms = current_time_millis(); + let metadata = match commit { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::TimedOut => return Err(HlsOriginResourceFetchError::Timeout), + Err(err) => return Err(HlsOriginResourceFetchError::cache_commit(&err)), + }; + let expires_at_ms = ready_at_ms + .saturating_add(context.cache_duration_ms) + .max(context.resource.expires_at_ms); + context.session.write().await.transient.mark_object_ready( + &context.cache_key, + content_type, + metadata.size, + ready_at_ms, + expires_at_ms, + ); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub fn hls_transient_origin_response( + response: reqwest::Response, + access: Arc, + origin_io_guard: Option, + media_activity_marker: Option, + now_ms: u64, + proxy_session_id: String, + resource_id: String, + resource_kind: TransientResourceKind, + origin_url: String, + origin_segment_timeout_ms: u64, +) -> axum::response::Response { + let mut builder = axum::response::Response::builder().status(response.status()); + for header_name in [ + header::CONTENT_TYPE, + header::CONTENT_LENGTH, + header::CONTENT_RANGE, + header::ACCEPT_RANGES, + header::CACHE_CONTROL, + header::ETAG, + header::LAST_MODIFIED, + ] { + if let Some(value) = response.headers().get(&header_name) { + builder = builder.header(header_name, value.clone()); + } + } + + let guard = HlsTransientReadGuard::new(access, now_ms); + let media_activity_guard = HlsTransientMediaActivityGuard::new(media_activity_marker, now_ms); + let deadline = hls_object_body_deadline(origin_segment_timeout_ms); + let stream = futures::stream::unfold( + ( + response.bytes_stream(), + Some(guard), + origin_io_guard, + Some(media_activity_guard), + Box::pin(sleep(hls_client_body_send_deadline())), + false, + ), + move |(mut stream, guard, origin_io_guard, media_activity_guard, mut send_deadline, finished)| { + let proxy_session_id = proxy_session_id.clone(); + let resource_id = resource_id.clone(); + let origin_url = origin_url.clone(); + async move { + if finished { + return None; + } + let next_chunk = tokio::select! { + () = send_deadline.as_mut() => { + log_hls_resource_timeout( + &proxy_session_id, + HlsResourceFetchLogContext { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: &resource_id, + origin_url: Some(&origin_url), + }, + HlsResourceFetchAttempt { attempt_index: 0, attempts: 1 }, + hls_client_body_send_deadline().as_millis(), + ); + return Some(( + Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")), + (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), + )); + } + next_chunk = tokio::time::timeout(deadline, stream.next()) => next_chunk, + }; + match next_chunk { + Ok(Some(Ok(chunk))) => { + refresh_hls_client_body_send_deadline(send_deadline.as_mut()); + Some((Ok(chunk), (stream, guard, origin_io_guard, media_activity_guard, send_deadline, false))) + } + Ok(Some(Err(err))) => Some(( + Err(io::Error::other(err)), + (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), + )), + Ok(None) => None, + Err(_) => { + log_hls_resource_timeout( + &proxy_session_id, + HlsResourceFetchLogContext { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: &resource_id, + origin_url: Some(&origin_url), + }, + HlsResourceFetchAttempt { attempt_index: 0, attempts: 1 }, + deadline.as_millis(), + ); + Some(( + Err(io::Error::new(io::ErrorKind::TimedOut, "transient passthrough body timed out")), + (stream, guard, origin_io_guard, media_activity_guard, send_deadline, true), + )) + } + } + } + }, + ); + try_unwrap_body!(builder.body(Body::from_stream(stream))) +} + +struct HlsTransientReadGuard { + access: Arc, +} + +impl HlsTransientReadGuard { + fn new(access: Arc, now_ms: u64) -> Self { + access.reader_started(now_ms); + Self { access } + } +} + +impl Drop for HlsTransientReadGuard { + fn drop(&mut self) { self.access.reader_finished(); } +} + +struct HlsTransientMediaActivityGuard { + marker: Option, +} + +impl HlsTransientMediaActivityGuard { + fn new(marker: Option, _now_ms: u64) -> Self { + Self { marker } + } +} + +impl Drop for HlsTransientMediaActivityGuard { + fn drop(&mut self) { + if let Some(marker) = &self.marker { + marker.spawn_mark_now(); + } + } +} + +pub struct HlsTransientOriginIoGuard { + session: HlsSessionHandle, + origin_io: HlsOriginIoContext, + lease_guard: Option, + started_generation: u64, +} + +impl HlsTransientOriginIoGuard { + pub fn new( + session: HlsSessionHandle, + origin_io: HlsOriginIoContext, + lease_guard: HlsOriginAccountIoLeaseGuard, + started_generation: u64, + ) -> Self { + Self { session, origin_io, lease_guard: Some(lease_guard), started_generation } + } +} + +impl Drop for HlsTransientOriginIoGuard { + fn drop(&mut self) { + let Some(lease_guard) = self.lease_guard.take() else { + return; + }; + let session = Arc::clone(&self.session); + let origin_io = self.origin_io.clone(); + let started_generation = self.started_generation; + tokio::spawn(async move { + let generation_valid = { + let mut session = session.write().await; + session.finish_origin_work(started_generation) + }; + let refresh_reservation = if generation_valid { + session.read().await.should_refresh_origin_reservation(chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default()) + } else { + false + }; + finish_hls_origin_account_io(&origin_io, &session, lease_guard, refresh_reservation).await; + let mut session = session.write().await; + if let Some(binding) = session.origin_account_binding.as_mut() { + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + binding.last_origin_io_at_ms = Some(now_ms); + if refresh_reservation { + binding.last_reservation_refresh_at_ms = Some(now_ms); + } + } + }); + } +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } diff --git a/backend/src/api/model/hls_provisioning.rs b/backend/src/api/model/hls_provisioning.rs new file mode 100644 index 000000000..5adf4ce8d --- /dev/null +++ b/backend/src/api/model/hls_provisioning.rs @@ -0,0 +1,797 @@ +use crate::{ + api::{ + model::{is_custom_video_stream_enabled, AppState, CustomVideoStreamType, HlsAccessLeaseId, ProxySessionId}, + panel_api::{can_provision_on_exhausted, try_provision_account_on_exhausted}, + }, + model::{ConfigInput, ProxyUserCredentials}, +}; +use axum::{ + body::Body, + http::{header, StatusCode}, + response::IntoResponse, +}; +use dashmap::DashMap; +use log::{debug, error}; +use shared::{ + utils::{sanitize_sensitive_info}, + defaults::{CUSTOM_VIDEO_PREFIX} +}; +use std::{ + fmt::Write as _, + sync::Arc, +}; + +const HLS_STATIC_CUSTOM_VIDEO_TARGET_DURATION_SECS: u64 = 10; +const HLS_STATIC_CUSTOM_VIDEO_EXTINF: &str = "10.0"; +const HLS_STATIC_CUSTOM_VIDEO_SEGMENT_COUNT: usize = 12; +const PROVISIONING_HLS_TARGET_DURATION_SECS: u64 = 2; +const PROVISIONING_HLS_EXTINF: &str = "2.000000"; +pub(crate) const CUSTOM_VIDEO_HLS_PROVISIONING_SEGMENT_COUNT: usize = + shared::defaults::PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT; +const HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS: u64 = 60_000; +const HLS_PROVISIONING_STALE_MARKER_MS: u64 = 5 * 60_000; +const HLS_CUSTOM_VIDEO_ROUTE_KIND: &str = "hls"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsProvisioningStatus { + InProgress, + Ready, + ProviderExhausted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct HlsProvisioningJobGroup { + last_started_at_ms: u64, + running_jobs: usize, + ready_slots: usize, + recent_failed_jobs: usize, + last_ready_at_ms: Option, + last_failed_at_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct HlsProvisioningConsumerKey { + input_name: Arc, + virtual_id: u32, +} + +impl HlsProvisioningConsumerKey { + fn new(input_name: Arc, virtual_id: u32) -> Self { Self { input_name, virtual_id } } +} + +#[derive(Debug, Clone)] +struct HlsProvisioningConsumer { + created_at_ms: u64, + last_seen_at_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct HlsProvisioningHandoffKey { + input_name: Arc, + virtual_id: u32, + proxy_session_id: Option, + access_lease_id: Option, +} + +impl HlsProvisioningHandoffKey { + fn new( + input_name: Arc, + virtual_id: u32, + proxy_session_id: Option<&ProxySessionId>, + access_lease_id: Option<&HlsAccessLeaseId>, + ) -> Self { + Self { + input_name, + virtual_id, + proxy_session_id: proxy_session_id.map(|id| id.0.clone()), + access_lease_id: access_lease_id.map(|id| id.0.clone()), + } + } +} + +#[derive(Debug, Default)] +pub struct HlsProvisioningState { + jobs: DashMap, HlsProvisioningJobGroup>, + consumers: DashMap, + handoffs: DashMap, +} + +impl HlsProvisioningState { + pub(in crate::api) fn new() -> Self { Self::default() } + + fn desired_jobs_for_waiting_consumers(consumer_count: usize) -> usize { + if consumer_count == 0 { + 0 + } else { + consumer_count.div_ceil(2) + } + } + + fn waiting_consumer_count(&self, input_name: &Arc, now_ms: u64) -> usize { + self.prune(now_ms); + self.consumers.iter().filter(|entry| &entry.key().input_name == input_name).count() + } + + fn start_jobs_for_waiting_consumers(&self, input_name: &Arc, now_ms: u64) -> usize { + let desired_jobs = Self::desired_jobs_for_waiting_consumers(self.waiting_consumer_count(input_name, now_ms)); + self.start_jobs_until(Arc::clone(input_name), desired_jobs, now_ms) + } + + fn start_jobs_until(&self, input_name: Arc, desired_jobs: usize, now_ms: u64) -> usize { + self.prune(now_ms); + if desired_jobs == 0 { + return 0; + } + if let Some(mut group) = self.jobs.get_mut(&input_name) { + let counted_jobs = group + .running_jobs + .saturating_add(group.ready_slots) + .saturating_add(group.recent_failed_jobs); + if counted_jobs >= desired_jobs { + return 0; + } + let jobs_to_start = desired_jobs - counted_jobs; + group.running_jobs = group.running_jobs.saturating_add(jobs_to_start); + group.last_started_at_ms = now_ms; + jobs_to_start + } else { + self.jobs.insert( + input_name, + HlsProvisioningJobGroup { + last_started_at_ms: now_ms, + running_jobs: desired_jobs, + ready_slots: 0, + recent_failed_jobs: 0, + last_ready_at_ms: None, + last_failed_at_ms: None, + }, + ); + desired_jobs + } + } + + fn mark_job_ready(&self, input_name: Arc, ready_at_ms: u64) { + if let Some(mut group) = self.jobs.get_mut(&input_name) { + group.running_jobs = group.running_jobs.saturating_sub(1); + group.ready_slots = group.ready_slots.saturating_add(1); + group.last_ready_at_ms = Some(ready_at_ms); + } else { + self.jobs.insert( + input_name, + HlsProvisioningJobGroup { + last_started_at_ms: ready_at_ms, + running_jobs: 0, + ready_slots: 1, + recent_failed_jobs: 0, + last_ready_at_ms: Some(ready_at_ms), + last_failed_at_ms: None, + }, + ); + } + } + + fn mark_job_provider_exhausted(&self, input_name: Arc, failed_at_ms: u64) { + if let Some(mut group) = self.jobs.get_mut(&input_name) { + group.running_jobs = group.running_jobs.saturating_sub(1); + group.recent_failed_jobs = group.recent_failed_jobs.saturating_add(1); + group.last_failed_at_ms = Some(failed_at_ms); + } else { + self.jobs.insert( + input_name, + HlsProvisioningJobGroup { + last_started_at_ms: failed_at_ms, + running_jobs: 0, + ready_slots: 0, + recent_failed_jobs: 1, + last_ready_at_ms: None, + last_failed_at_ms: Some(failed_at_ms), + }, + ); + } + } + + pub(in crate::api) fn touch_consumer(&self, input_name: Arc, virtual_id: u32, now_ms: u64) { + self.prune(now_ms); + let key = HlsProvisioningConsumerKey::new(input_name, virtual_id); + if let Some(mut consumer) = self.consumers.get_mut(&key) { + consumer.last_seen_at_ms = now_ms; + } else { + self.consumers.insert(key, HlsProvisioningConsumer { created_at_ms: now_ms, last_seen_at_ms: now_ms }); + } + } + + fn job_status(&self, input_name: &Arc, now_ms: u64) -> Option { + self.prune(now_ms); + let group = self.jobs.get(input_name)?; + if group.ready_slots > 0 { + return Some(HlsProvisioningStatus::Ready); + } + if group.running_jobs > 0 { + return Some(HlsProvisioningStatus::InProgress); + } + if group.recent_failed_jobs > 0 { + return Some(HlsProvisioningStatus::ProviderExhausted); + } + None + } + + pub(in crate::api) fn consumer_status( + &self, + input_name: &Arc, + virtual_id: u32, + now_ms: u64, + ) -> Option { + self.prune(now_ms); + let key = HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id); + self.consumers.get(&key)?; + self.job_status(input_name, now_ms) + } + + pub(in crate::api) fn has_consumer(&self, input_name: &Arc, virtual_id: u32, now_ms: u64) -> bool { + self.prune(now_ms); + let key = HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id); + self.consumers.contains_key(&key) + } + + pub(in crate::api) fn clear_consumer(&self, input_name: &Arc, virtual_id: u32) { + self.consumers + .remove(&HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id)); + } + + pub(in crate::api) fn mark_handoff_once( + &self, + input_name: &Arc, + virtual_id: u32, + proxy_session_id: Option<&ProxySessionId>, + access_lease_id: Option<&HlsAccessLeaseId>, + now_ms: u64, + ) -> bool { + self.prune(now_ms); + let key = HlsProvisioningHandoffKey::new(Arc::clone(input_name), virtual_id, proxy_session_id, access_lease_id); + if self.handoffs.contains_key(&key) { + return false; + } + self.handoffs.insert(key, now_ms); + true + } + + pub(in crate::api) fn take_ready_slot_for_consumer( + &self, + input_name: &Arc, + virtual_id: u32, + now_ms: u64, + ) -> bool { + self.prune(now_ms); + let key = HlsProvisioningConsumerKey::new(Arc::clone(input_name), virtual_id); + if !self.consumers.contains_key(&key) { + return false; + } + let Some(mut group) = self.jobs.get_mut(input_name) else { + return false; + }; + if group.ready_slots == 0 { + return false; + } + group.ready_slots -= 1; + drop(group); + self.consumers.remove(&key); + true + } + + fn prune(&self, now_ms: u64) { + self.jobs.retain(|_, group| { + if let Some(ready_at_ms) = group.last_ready_at_ms { + if ready_at_ms.saturating_add(HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS) < now_ms { + group.ready_slots = 0; + group.last_ready_at_ms = None; + } + } + if let Some(failed_at_ms) = group.last_failed_at_ms { + if failed_at_ms.saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) < now_ms { + group.recent_failed_jobs = 0; + group.last_failed_at_ms = None; + } + } + let latest_reference_ms = [ + Some(group.last_started_at_ms), + group.last_ready_at_ms, + group.last_failed_at_ms, + ] + .into_iter() + .flatten() + .max() + .unwrap_or_default(); + group.running_jobs > 0 + || group.ready_slots > 0 + || group.recent_failed_jobs > 0 + || latest_reference_ms.saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) >= now_ms + }); + self.consumers.retain(|_, consumer| { + consumer + .last_seen_at_ms + .max(consumer.created_at_ms) + .saturating_add(HLS_PROVISIONING_STALE_MARKER_MS) + >= now_ms + }); + self.handoffs.retain(|_, marked_at_ms| { + marked_at_ms.saturating_add(HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS) >= now_ms + }); + } +} + +fn current_time_millis() -> u64 { chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default() } + +fn hls_response(hls_content: String) -> axum::response::Response { + match axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/vnd.apple.mpegurl") + .header(header::CACHE_CONTROL, "no-store, no-cache, must-revalidate") + .body(Body::from(hls_content)) + { + Ok(response) => response, + Err(err) => { + error!("Failed to build HLS response: {}", sanitize_sensitive_info(err.to_string().as_str())); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } +} + +pub(crate) fn hls_custom_video_type_configured(app_state: &Arc, video_type: CustomVideoStreamType) -> bool { + if !is_custom_video_stream_enabled(&app_state.app_config) { + return false; + } + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + match video_type { + CustomVideoStreamType::ChannelUnavailable => { + custom_stream_response.as_ref().and_then(|response| response.channel_unavailable.as_ref()).is_some() + } + CustomVideoStreamType::UserConnectionsExhausted => { + custom_stream_response.as_ref().and_then(|response| response.user_connections_exhausted.as_ref()).is_some() + } + CustomVideoStreamType::ProviderConnectionsExhausted => custom_stream_response + .as_ref() + .and_then(|response| response.provider_connections_exhausted.as_ref()) + .is_some(), + CustomVideoStreamType::LowPriorityPreempted => { + custom_stream_response.as_ref().and_then(|response| response.low_priority_preempted.as_ref()).is_some() + } + CustomVideoStreamType::UserAccountExpired => { + custom_stream_response.as_ref().and_then(|response| response.user_account_expired.as_ref()).is_some() + } + CustomVideoStreamType::Provisioning => { + custom_stream_response.as_ref().and_then(|response| response.panel_api_provisioning.as_ref()).is_some() + } + CustomVideoStreamType::HlsSessionOrLeaseExpired => custom_stream_response + .as_ref() + .and_then(|response| response.hls_session_or_lease_expired.as_ref()) + .is_some(), + } +} + +fn hls_custom_video_url(base_url: &str, user: &ProxyUserCredentials, video_type: CustomVideoStreamType) -> String { + format!( + "{}/{CUSTOM_VIDEO_PREFIX}/{HLS_CUSTOM_VIDEO_ROUTE_KIND}/{}/{}/{}.ts", + base_url.trim_end_matches('/'), + user.username, + user.password, + video_type + ) +} + +pub(crate) fn hls_custom_video_manifest_path( + user: &ProxyUserCredentials, + video_type: CustomVideoStreamType, +) -> String { + format!( + "/{CUSTOM_VIDEO_PREFIX}/{HLS_CUSTOM_VIDEO_ROUTE_KIND}/{}/{}/{}.m3u8", + user.username, user.password, video_type + ) +} + +pub(crate) fn hls_panel_provisioning_segment_route_name(index: usize) -> String { + format!("provisioning_{index:03}.ts") +} + +pub(crate) fn parse_hls_panel_provisioning_segment_route_name(stream_type: &str) -> Option { + let raw = stream_type.strip_suffix(".ts").unwrap_or(stream_type); + let index = raw.strip_prefix("provisioning_")?.parse::().ok()?; + (index < CUSTOM_VIDEO_HLS_PROVISIONING_SEGMENT_COUNT).then_some(index) +} + +fn hls_panel_provisioning_segment_url(base_url: &str, user: &ProxyUserCredentials, index: usize) -> String { + format!( + "{}/{CUSTOM_VIDEO_PREFIX}/{HLS_CUSTOM_VIDEO_ROUTE_KIND}/{}/{}/{}", + base_url.trim_end_matches('/'), + user.username, + user.password, + hls_panel_provisioning_segment_route_name(index) + ) +} + +pub(crate) fn hls_panel_provisioning_manifest_path(user: &ProxyUserCredentials, virtual_id: u32) -> String { + format!( + "/{CUSTOM_VIDEO_PREFIX}/{HLS_CUSTOM_VIDEO_ROUTE_KIND}/{}/{}/provisioning.m3u8?id={virtual_id}", + user.username, user.password + ) +} + +pub(crate) fn hls_provisioning_discontinuity_sequence(_now_ms: u64) -> u64 { + 0 +} + +fn build_hls_panel_provisioning_manifest_body(mut segment_url: impl FnMut(usize) -> String) -> String { + let media_sequence = 0; + let mut playlist = format!( + "#EXTM3U\n\ + #EXT-X-VERSION:3\n\ + #EXT-X-TARGETDURATION:{PROVISIONING_HLS_TARGET_DURATION_SECS}\n\ + #EXT-X-MEDIA-SEQUENCE:{media_sequence}\n\ + #EXT-X-INDEPENDENT-SEGMENTS\n" + ); + for index in 0..CUSTOM_VIDEO_HLS_PROVISIONING_SEGMENT_COUNT { + let video_url = segment_url(index); + let _ = write!(playlist, "#EXTINF:{PROVISIONING_HLS_EXTINF},\n{video_url}\n"); + } + playlist +} + +fn build_hls_static_custom_video_manifest_body(video_url: &str) -> String { + let mut playlist = format!( + "#EXTM3U\n\ + #EXT-X-VERSION:3\n\ + #EXT-X-TARGETDURATION:{HLS_STATIC_CUSTOM_VIDEO_TARGET_DURATION_SECS}\n\ + #EXT-X-MEDIA-SEQUENCE:0\n" + ); + for _ in 0..HLS_STATIC_CUSTOM_VIDEO_SEGMENT_COUNT { + let _ = write!(playlist, "#EXTINF:{HLS_STATIC_CUSTOM_VIDEO_EXTINF},\n{video_url}\n"); + } + playlist.push_str("#EXT-X-ENDLIST\n"); + playlist +} + +pub(crate) fn build_hls_custom_video_manifest_body( + base_url: &str, + user: &ProxyUserCredentials, + video_type: CustomVideoStreamType, + _now_ms: u64, + _virtual_id: Option, +) -> String { + if video_type != CustomVideoStreamType::Provisioning { + let video_url = hls_custom_video_url(base_url, user, video_type); + return build_hls_static_custom_video_manifest_body(&video_url); + } + + build_hls_panel_provisioning_manifest_body(|index| hls_panel_provisioning_segment_url(base_url, user, index)) +} + +pub(crate) fn hls_custom_video_manifest_response_with_virtual_id( + app_state: &Arc, + user: &ProxyUserCredentials, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, + virtual_id: Option, +) -> axum::response::Response { + if !hls_custom_video_type_configured(app_state, video_type) { + return fallback_status.into_response(); + } + let Some(server_info) = app_state.app_config.get_user_server_info(user) else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + + hls_response(build_hls_custom_video_manifest_body( + &server_info.get_base_url(), + user, + video_type, + current_time_millis(), + virtual_id, + )) +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct HlsPanelProvisioningRedirectPaths<'a> { + pub waiting_manifest_path: Option<&'a str>, +} + +pub(crate) async fn try_hls_panel_provisioning_manifest_response( + app_state: &Arc, + user: &ProxyUserCredentials, + input: &ConfigInput, + virtual_id: u32, + redirect_paths: HlsPanelProvisioningRedirectPaths<'_>, + server_path: Option<&str>, + fallback_status: StatusCode, +) -> Option { + let now_ms = current_time_millis(); + app_state + .hls_provisioning + .touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); + if provisioning_enabled { + start_hls_panel_provisioning_once(app_state, input); + } + let status = if let Some(status) = app_state + .hls_provisioning + .consumer_status(&input.name, virtual_id, now_ms) + { + status + } else if provisioning_enabled { + HlsProvisioningStatus::InProgress + } else { + return None; + }; + + match status { + HlsProvisioningStatus::InProgress | HlsProvisioningStatus::Ready => { + // A ready provisioning job only means that credentials were persisted/probed. + // Stream handoff must happen in a route that can reserve a runtime origin account first. + if !hls_custom_video_type_configured(app_state, CustomVideoStreamType::Provisioning) { + return Some(fallback_status.into_response()); + } + let server_path = + server_path.map(str::to_string).or_else(|| app_state.app_config.get_user_server_info(user).and_then(|server| server.path)); + let manifest_path = + redirect_paths.waiting_manifest_path.map_or_else(|| hls_panel_provisioning_manifest_path(user, virtual_id), str::to_string); + Some(hls_virtual_entry_redirect_response( + &manifest_path, + server_path.as_deref(), + )) + } + HlsProvisioningStatus::ProviderExhausted => Some(hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::ProviderConnectionsExhausted, + fallback_status, + Some(virtual_id), + )), + } +} + +pub(crate) fn start_hls_panel_provisioning_once(app_state: &Arc, input: &ConfigInput) -> bool { + if !can_provision_on_exhausted(app_state.as_ref(), input) { + return false; + } + let key = Arc::clone(&input.name); + let now_ms = current_time_millis(); + let jobs_to_start = app_state + .hls_provisioning + .start_jobs_for_waiting_consumers(&key, now_ms); + if jobs_to_start == 0 { + return false; + } + for job_index in 0..jobs_to_start { + let app_state = Arc::clone(app_state); + let key = Arc::clone(&key); + tokio::spawn(async move { + debug!( + "HLS panel provisioning started: input={} job_index={}", + sanitize_sensitive_info(key.as_ref()), + job_index + ); + let outcome = try_provision_account_on_exhausted(&app_state, &key).await; + let ready = outcome.is_some(); + let finished_at_ms = current_time_millis(); + if ready { + app_state.hls_provisioning.mark_job_ready(Arc::clone(&key), finished_at_ms); + } else { + app_state + .hls_provisioning + .mark_job_provider_exhausted(Arc::clone(&key), finished_at_ms); + } + debug!( + "HLS panel provisioning completed: input={} outcome={} ready={}", + sanitize_sensitive_info(key.as_ref()), + outcome.as_ref().map_or("unchanged", |outcome| outcome.kind_label()), + ready + ); + }); + } + true +} + +fn prefixed_hls_entry_path(original_hls_entry_path: &str, server_path: Option<&str>) -> String { + let path = if original_hls_entry_path.starts_with('/') { + original_hls_entry_path.to_string() + } else { + format!("/{original_hls_entry_path}") + }; + let Some(server_path) = server_path.map(str::trim).filter(|path| !path.is_empty() && *path != "/") else { + return path; + }; + format!("/{}/{}", server_path.trim_matches('/'), path.trim_start_matches('/')) +} + +pub(crate) fn hls_virtual_entry_redirect_response( + original_hls_entry_path: &str, + server_path: Option<&str>, +) -> axum::response::Response { + match axum::response::Response::builder() + .status(StatusCode::TEMPORARY_REDIRECT) + .header(header::LOCATION, prefixed_hls_entry_path(original_hls_entry_path, server_path)) + .header(header::CACHE_CONTROL, "no-store") + .body(Body::empty()) + { + Ok(response) => response, + Err(err) => { + error!("Failed to build HLS provisioning redirect: {}", sanitize_sensitive_info(err.to_string().as_str())); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input_name() -> Arc { Arc::::from("cdn-test") } + + #[test] + fn hls_provisioning_state_tracks_in_progress_ready_and_provider_exhausted() { + let state = HlsProvisioningState::new(); + let input = input_name(); + + state.touch_consumer(Arc::clone(&input), 57, 900); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_000), 1); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_100), 0); + assert_eq!(state.job_status(&input, 1_200), Some(HlsProvisioningStatus::InProgress)); + + state.mark_job_ready(Arc::clone(&input), 2_000); + assert_eq!(state.job_status(&input, 2_100), Some(HlsProvisioningStatus::Ready)); + assert!(state.take_ready_slot_for_consumer(&input, 57, 2_200)); + + state.touch_consumer(Arc::clone(&input), 59, 2_900); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 2_950), 1); + state.mark_job_provider_exhausted(Arc::clone(&input), 3_000); + assert_eq!(state.job_status(&input, 3_100), Some(HlsProvisioningStatus::ProviderExhausted)); + } + + #[test] + fn hls_provisioning_ready_expires_after_redirect_window() { + let state = HlsProvisioningState::new(); + let input = input_name(); + let ready_at_ms = 10_000; + + state.mark_job_ready(Arc::clone(&input), ready_at_ms); + assert_eq!( + state.job_status(&input, ready_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS), + Some(HlsProvisioningStatus::Ready) + ); + assert_eq!( + state.job_status(&input, ready_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS + 1), + None + ); + } + + #[test] + fn hls_provisioning_handoff_marker_is_idempotent_per_shared_session_lease() { + let state = HlsProvisioningState::new(); + let input = input_name(); + let proxy_session_id = ProxySessionId("shared-session".to_string()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + + assert!(state.mark_handoff_once(&input, 57, Some(&proxy_session_id), Some(&access_lease_id), 1_000)); + assert!(!state.mark_handoff_once(&input, 57, Some(&proxy_session_id), Some(&access_lease_id), 1_100)); + + let next_access_lease_id = HlsAccessLeaseId("next-access-lease".to_string()); + assert!(state.mark_handoff_once( + &input, + 57, + Some(&proxy_session_id), + Some(&next_access_lease_id), + 1_200 + )); + } + + #[test] + fn hls_provisioning_handoff_marker_expires_after_redirect_window() { + let state = HlsProvisioningState::new(); + let input = input_name(); + let proxy_session_id = ProxySessionId("shared-session".to_string()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let marked_at_ms = 10_000; + + assert!(state.mark_handoff_once( + &input, + 57, + Some(&proxy_session_id), + Some(&access_lease_id), + marked_at_ms + )); + assert!(!state.mark_handoff_once( + &input, + 57, + Some(&proxy_session_id), + Some(&access_lease_id), + marked_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS + )); + assert!(state.mark_handoff_once( + &input, + 57, + Some(&proxy_session_id), + Some(&access_lease_id), + marked_at_ms + HLS_PROVISIONING_COMPLETED_REDIRECT_WINDOW_MS + 1 + )); + } + + #[test] + fn hls_provisioning_segment_route_names_are_bounded() { + assert_eq!(hls_panel_provisioning_segment_route_name(0), "provisioning_000.ts"); + assert_eq!(hls_panel_provisioning_segment_route_name(5), "provisioning_005.ts"); + assert_eq!(parse_hls_panel_provisioning_segment_route_name("provisioning_000.ts"), Some(0)); + assert_eq!(parse_hls_panel_provisioning_segment_route_name("provisioning_005.ts"), Some(5)); + assert_eq!(parse_hls_panel_provisioning_segment_route_name("provisioning_006.ts"), None); + assert_eq!(parse_hls_panel_provisioning_segment_route_name("provisioning.ts"), None); + } + + #[test] + fn hls_provisioning_consumers_are_tracked_per_virtual_id() { + let state = HlsProvisioningState::new(); + let input = input_name(); + + state.touch_consumer(Arc::clone(&input), 57, 1_100); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_150), 1); + + assert_eq!(state.consumer_status(&input, 57, 1_200), Some(HlsProvisioningStatus::InProgress)); + assert_eq!(state.consumer_status(&input, 59, 1_200), None); + + state.touch_consumer(Arc::clone(&input), 59, 1_300); + state.mark_job_ready(Arc::clone(&input), 2_000); + assert_eq!(state.consumer_status(&input, 57, 2_100), Some(HlsProvisioningStatus::Ready)); + assert_eq!(state.consumer_status(&input, 59, 2_100), Some(HlsProvisioningStatus::Ready)); + + assert!(state.take_ready_slot_for_consumer(&input, 59, 2_150)); + assert_eq!(state.consumer_status(&input, 59, 2_200), None); + assert_eq!(state.consumer_status(&input, 57, 2_200), None); + assert_eq!(state.job_status(&input, 2_200), None); + } + + #[test] + fn hls_provisioning_stale_provider_exhausted_job_can_restart() { + let state = HlsProvisioningState::new(); + let input = input_name(); + + state.touch_consumer(Arc::clone(&input), 57, 900); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_000), 1); + state.mark_job_provider_exhausted(Arc::clone(&input), 2_000); + assert_eq!(state.job_status(&input, 2_100), Some(HlsProvisioningStatus::ProviderExhausted)); + + let after_stale = 2_000 + HLS_PROVISIONING_STALE_MARKER_MS + 1; + assert_eq!(state.job_status(&input, after_stale), None); + state.touch_consumer(Arc::clone(&input), 57, after_stale + 1); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, after_stale + 2), 1); + } + + #[test] + fn hls_provisioning_starts_one_job_for_each_two_waiting_consumers() { + let state = HlsProvisioningState::new(); + let input = input_name(); + + state.touch_consumer(Arc::clone(&input), 57, 1_000); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_010), 1); + + state.touch_consumer(Arc::clone(&input), 59, 1_100); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_110), 0); + + state.touch_consumer(Arc::clone(&input), 61, 1_200); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_210), 1); + + state.touch_consumer(Arc::clone(&input), 63, 1_300); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_310), 0); + } + + #[test] + fn hls_provisioning_recent_failures_count_against_backpressure_target() { + let state = HlsProvisioningState::new(); + let input = input_name(); + + state.touch_consumer(Arc::clone(&input), 57, 1_000); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 1_010), 1); + state.mark_job_provider_exhausted(Arc::clone(&input), 2_000); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 2_010), 0); + + state.touch_consumer(Arc::clone(&input), 59, 2_100); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 2_110), 0); + + state.touch_consumer(Arc::clone(&input), 61, 2_200); + assert_eq!(state.start_jobs_for_waiting_consumers(&input, 2_210), 1); + } +} diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index 6793e55bb..eb9d9e6d8 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -18,7 +18,6 @@ use dashmap::{mapref::entry::Entry, DashMap}; use log::{debug, error, info, warn}; use parking_lot::Mutex as ParkingMutex; use serde::{Deserialize, Serialize}; -use shared::utils::default_probe_user_priority; use shared::{ create_bitset, error::TuliproxError, @@ -27,6 +26,7 @@ use shared::{ VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, }, utils::generate_provider_playlist_uuid, + defaults::default_probe_user_priority, }; use std::{ cmp::min, @@ -1031,7 +1031,7 @@ impl MetadataUpdateManager { // Lock-free admission with CAS: reserve one queue slot only if capacity allows. if pending_task_count - .fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| { + .try_update(Ordering::AcqRel, Ordering::Relaxed, |current| { if current < max_queue_size { Some(current + 1) } else { @@ -1081,7 +1081,7 @@ impl MetadataUpdateManager { #[inline] fn decrement_pending_task_count(pending_task_count: &AtomicUsize) { // Guard against accidental underflow in edge/error paths. - let _ = pending_task_count.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| current.checked_sub(1)); + let _ = pending_task_count.try_update(Ordering::AcqRel, Ordering::Relaxed, |current| current.checked_sub(1)); } fn merge_task_payload(existing: &mut UpdateTask, task: UpdateTask) -> bool { diff --git a/backend/src/api/model/mod.rs b/backend/src/api/model/mod.rs index 4f68ca73a..436f78e8f 100644 --- a/backend/src/api/model/mod.rs +++ b/backend/src/api/model/mod.rs @@ -5,12 +5,15 @@ mod app_state; mod connection_manager; mod download; mod event_manager; +mod hls_cache; +mod hls_provisioning; mod metadata_update_manager; mod model_utils; mod playlist_mem_cache; mod provider_config; mod provider_dns_manager; mod provider_lineup_manager; +mod proxy; mod qos_aggregation_manager; mod recording_worker; mod request; @@ -22,15 +25,32 @@ mod xtream; pub(crate) use self::streams::*; pub use self::{ - active_provider_manager::*, app_state::*, connection_manager::*, event_manager::*, metadata_update_manager::*, - playlist_mem_cache::*, provider_dns_manager::*, provider_lineup_manager::*, stream::*, update_guard::*, + active_provider_manager::*, app_state::*, connection_manager::*, event_manager::*, hls_cache::*, + hls_provisioning::HlsProvisioningState, metadata_update_manager::*, playlist_mem_cache::*, provider_dns_manager::*, + provider_lineup_manager::*, proxy::*, stream::*, update_guard::*, }; pub(in crate::api) use self::{ - active_user_manager::*, download::*, model_utils::*, provider_config::*, qos_aggregation_manager::*, - recording_worker::*, request::*, stream_error::*, xtream::*, + active_user_manager::*, + admission_strategy::{evaluate_strategy, AdmissionDecision, EvictionCandidate, GraceMode, StrategyContext}, + download::*, + hls_provisioning::{ + hls_custom_video_manifest_path, hls_custom_video_manifest_response_with_virtual_id, + hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, + parse_hls_panel_provisioning_segment_route_name, start_hls_panel_provisioning_once, + try_hls_panel_provisioning_manifest_response, HlsPanelProvisioningRedirectPaths, + HlsProvisioningStatus, + }, + model_utils::*, + provider_config::*, + qos_aggregation_manager::*, + recording_worker::*, + request::*, + stream_error::*, + xtream::*, }; -pub(in crate::api) use self::admission_strategy::{ - AdmissionDecision, EvictionCandidate, GraceMode, StrategyContext, evaluate_strategy, +#[cfg(test)] +pub(in crate::api) use self::hls_provisioning::{ + build_hls_custom_video_manifest_body, hls_panel_provisioning_manifest_path, }; mod batch_result_collector; pub use self::batch_result_collector::*; diff --git a/backend/src/api/model/provider_config.rs b/backend/src/api/model/provider_config.rs index 76c92d955..cebff753d 100644 --- a/backend/src/api/model/provider_config.rs +++ b/backend/src/api/model/provider_config.rs @@ -340,6 +340,8 @@ impl fmt::Display for ProviderConfigWrapper { impl ProviderConfigWrapper { pub fn new(cfg: ProviderConfig) -> Self { Self { inner: Arc::new(cfg) } } + pub(in crate::api::model) fn config(&self) -> Arc { Arc::clone(&self.inner) } + pub async fn force_allocate(&self) -> ProviderAllocation { if self.inner.force_allocate().await { ProviderAllocation::new_available(Arc::clone(&self.inner)) diff --git a/backend/src/api/model/provider_lineup_manager.rs b/backend/src/api/model/provider_lineup_manager.rs index 71c14fdd3..df38681ba 100644 --- a/backend/src/api/model/provider_lineup_manager.rs +++ b/backend/src/api/model/provider_lineup_manager.rs @@ -887,6 +887,11 @@ impl ProviderLineupManager { } } + pub fn find_provider_config(&self, provider_name: &Arc) -> Option> { + let snapshot = self.snapshot.load_full(); + Self::get_provider_config_by_name(provider_name, &snapshot.providers).map(|(_, config)| config.config()) + } + pub fn provider_names_for_input(&self, input_name: &Arc) -> Vec> { let snapshot = self.snapshot.load_full(); Self::get_provider_config_by_name(input_name, &snapshot.providers) diff --git a/backend/src/api/model/proxy/header_policy.rs b/backend/src/api/model/proxy/header_policy.rs new file mode 100644 index 000000000..b96d01713 --- /dev/null +++ b/backend/src/api/model/proxy/header_policy.rs @@ -0,0 +1,142 @@ +//! Shared hop-by-hop header policy for the HLS cache proxy and the MPEG-TS reverse-proxy. +//! +//! Both pipelines strip the same security-sensitive headers before forwarding to the +//! origin; before this module existed the HLS cache hard-coded its own list +//! (`should_remove_hls_origin_header`) that drifted from the `ReverseProxyDisabledHeaderConfig` +//! path used by the MPEG-TS reverse proxy. Adding a new hop-by-hop default required +//! editing two files; let a future maintainer forget one and the operator's +//! "disabled header" config silently leaks on the HLS path. +//! +//! The single source of truth here feeds both call sites via `HopByHopHeader::is_sensitive`. + +use crate::model::ReverseProxyDisabledHeaderConfig; + +/// Protocol family a header policy is being applied for. +/// +/// Today only `Hls` ships with hard-coded defaults; `MpegTs` defers entirely to the +/// operator-configured `ReverseProxyDisabledHeaderConfig`. The enum exists so future +/// asymmetric rules (e.g. `Sec-Fetch-*` for HLS, `Cookie2` for legacy MPEG-TS providers) +/// have a single landing point. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HeaderProtocol { + Hls, + MpegTs, +} + +/// Static, hard-coded list of headers whose presence is *always* a security problem on +/// outbound origin requests: transport-layer framing (`Connection`, `Transfer-Encoding`, +/// `Upgrade`), authentication credentials (`Authorization`, `Cookie`, `Proxy-Authorization`), +/// and Tuliprox internal markers (`x-tuliprox-*`). +/// +/// The list does NOT depend on operator config; the config only adds *additional* +/// removals via `ReverseProxyDisabledHeaderConfig::should_remove`. +const ALWAYS_SENSITIVE: &[&str] = &[ + "authorization", + "connection", + "cookie", + "cookie2", + "host", + "proxy-authorization", + "set-cookie", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", +]; + +/// Hard-coded Tuliprox internal header prefix — never forwarded to origin regardless +/// of config. Matches `x-tuliprox-*`. +const TULIPROX_HEADER_PREFIX: &str = "x-tuliprox-"; + +/// Single hop-by-hop header policy shared between HLS and MPEG-TS proxy paths. +/// +/// `is_sensitive` returns true when the header must be stripped before sending the +/// request to the origin. The operator-configured `ReverseProxyDisabledHeaderConfig` +/// is layered on top of the hard-coded defaults so adding a new "disabled header" +/// config applies to both protocols uniformly. +pub struct HopByHopHeader; + +impl HopByHopHeader { + /// True when a header must never be forwarded to the origin for the given protocol + /// family. Combines: + /// 1. hard-coded hop-by-hop + Tuliprox-internal headers (always sensitive) + /// 2. operator-configured disabled-header list, when supplied + /// + /// The protocol-family parameter is reserved for future asymmetric rules. Today both + /// `Hls` and `MpegTs` share the same set; switching on it costs nothing today and + /// gives the next maintainer one obvious place to extend. + pub fn is_sensitive( + header_name: &str, + protocol: HeaderProtocol, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + ) -> bool { + let _ = protocol; // reserved for future per-protocol rules + let header_lc = header_name.trim().to_ascii_lowercase(); + ALWAYS_SENSITIVE.contains(&header_lc.as_str()) + || header_lc.starts_with(TULIPROX_HEADER_PREFIX) + || disabled_headers.is_some_and(|disabled| disabled.should_remove(header_lc.as_str())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn disabled() -> ReverseProxyDisabledHeaderConfig { + ReverseProxyDisabledHeaderConfig { + referer_header: true, + x_header: true, + cloudflare_header: true, + custom_header: vec!["X-Origin-Secret".to_string()], + } + } + + #[test] + fn strips_hardcoded_hop_by_hop_for_both_protocols() { + for header in ["Authorization", "Cookie", "Connection", "TE", "Trailer", "Transfer-Encoding", + "Upgrade", "Proxy-Authorization", "Host", "Set-Cookie"] { + assert!( + HopByHopHeader::is_sensitive(header, HeaderProtocol::Hls, None), + "Hls should drop hard-coded {header}" + ); + assert!( + HopByHopHeader::is_sensitive(header, HeaderProtocol::MpegTs, None), + "MpegTs should drop hard-coded {header}" + ); + } + } + + #[test] + fn strips_tuliprox_internal_prefix() { + assert!(HopByHopHeader::is_sensitive("X-Tuliprox-Main-Revision", HeaderProtocol::Hls, None)); + assert!(HopByHopHeader::is_sensitive("x-tuliprox-debug", HeaderProtocol::MpegTs, None)); + } + + #[test] + fn operator_config_layers_on_top_of_hardcoded_list() { + let d = disabled(); + assert!(HopByHopHeader::is_sensitive("Referer", HeaderProtocol::Hls, Some(&d))); + assert!(HopByHopHeader::is_sensitive("X-Blocked", HeaderProtocol::Hls, Some(&d))); + assert!(HopByHopHeader::is_sensitive("CF-Ray", HeaderProtocol::Hls, Some(&d))); + assert!(HopByHopHeader::is_sensitive("x-origin-secret", HeaderProtocol::Hls, Some(&d))); + } + + #[test] + fn passthrough_headers_are_not_sensitive() { + assert!(!HopByHopHeader::is_sensitive("Accept-Language", HeaderProtocol::Hls, None)); + assert!(!HopByHopHeader::is_sensitive("Accept-Encoding", HeaderProtocol::Hls, None)); + assert!(!HopByHopHeader::is_sensitive("Content-Type", HeaderProtocol::MpegTs, None)); + } + + #[test] + fn mpegts_protocol_path_with_operator_config() { + // Without operator config, MpegTs falls through to the same hard-coded defaults. + assert!(HopByHopHeader::is_sensitive("Authorization", HeaderProtocol::MpegTs, None)); + assert!(!HopByHopHeader::is_sensitive("X-Blocked", HeaderProtocol::MpegTs, None)); + + // With operator config, the operator's list is honored. + let d = disabled(); + assert!(HopByHopHeader::is_sensitive("X-Blocked", HeaderProtocol::MpegTs, Some(&d))); + } +} diff --git a/backend/src/api/model/proxy/mod.rs b/backend/src/api/model/proxy/mod.rs new file mode 100644 index 000000000..cae1a98e0 --- /dev/null +++ b/backend/src/api/model/proxy/mod.rs @@ -0,0 +1,7 @@ +//! Cross-proxy shared helpers (header policy, request building, etc.). +//! +//! Lives next to the existing `api::model::*` modules and is intentionally small. +//! `header_policy` is the canonical home for hop-by-hop header rules used by both +//! the HLS cache proxy and the MPEG-TS reverse proxy — see its module docs. + +pub mod header_policy; diff --git a/backend/src/api/model/qos_aggregation_manager.rs b/backend/src/api/model/qos_aggregation_manager.rs index 161c15dea..2299a3faf 100644 --- a/backend/src/api/model/qos_aggregation_manager.rs +++ b/backend/src/api/model/qos_aggregation_manager.rs @@ -806,6 +806,7 @@ mod tests { stream_history_enabled: false, ..Default::default() }), + hls_cache: None, qos_aggregation: Some(QosAggregationConfig { enabled: true, interval_secs: 300, diff --git a/backend/src/api/model/stream.rs b/backend/src/api/model/stream.rs index 46384275f..2c97bfd17 100644 --- a/backend/src/api/model/stream.rs +++ b/backend/src/api/model/stream.rs @@ -5,6 +5,7 @@ use crate::{ use axum::http::StatusCode; use bytes::Bytes; use futures::stream::BoxStream; +use shared::model::{PlaylistItemType, StreamChannel}; use std::{collections::HashMap, sync::Arc}; use tokio_util::sync::CancellationToken; use url::Url; @@ -17,11 +18,31 @@ pub type ProviderStreamResponse = (Option, ProviderStreamIn pub type ProviderStreamFactoryResponse = (BoxedProviderStream, ProviderStreamInfo); +pub(crate) fn uses_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { + !stream_channel.shared + && matches!( + stream_channel.item_type, + PlaylistItemType::Video + | PlaylistItemType::Series + | PlaylistItemType::LocalVideo + | PlaylistItemType::LocalSeries + ) +} + type StreamUrl = Arc; type ProviderName = Arc; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProviderStreamCustomReason { + ProviderExhausted, + UnmappedProviderUrl, +} + pub enum ProviderStreamState { - Custom(ProviderStreamResponse), + Custom { + response: ProviderStreamResponse, + reason: ProviderStreamCustomReason, + }, Available(Option, StreamUrl), GracePeriod(Option, StreamUrl), } diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index 2720c91e9..b1c735374 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -2,11 +2,11 @@ use crate::{ api::{ api_utils::get_stream_options, model::{ + AppState, BoxedProviderStream, CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, + MeteringStream, PendingProviderWakeSource, ProviderHandle, ProviderStreamFactoryOptions, StreamDetails, + StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer, connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, - create_provider_stream, AppState, BoxedProviderStream, CleanupEvent, ConnectionManager, - CustomVideoStreamType, EventManager, MeteringStream, PendingProviderWakeSource, ProviderHandle, - ProviderStreamFactoryOptions, StreamDetails, StreamError, StreamMeterHandle, TimedClientStream, - TransportStreamBuffer, + create_provider_stream, uses_direct_body_idle_timeout, }, panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, }, @@ -14,10 +14,10 @@ use crate::{ model::{ConfigInput, ProxyUserCredentials}, utils::debug_if_enabled, }; -use axum::http::{header::USER_AGENT, HeaderMap}; +use axum::http::{HeaderMap, header::USER_AGENT}; use bytes::Bytes; -use futures::{task::AtomicWaker, Future, Stream, StreamExt}; -use log::{error, info}; +use futures::{Future, Stream, StreamExt, task::AtomicWaker}; +use log::{error, info, warn}; use shared::model::FailureStage; use shared::utils::Internable; use shared::{ @@ -28,14 +28,18 @@ use std::{ net::SocketAddr, pin::Pin, sync::{ - atomic::{AtomicU8, Ordering}, Arc, + atomic::{AtomicU8, Ordering}, }, task::{Context, Poll}, }; use tokio::sync::Notify; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +const DIRECT_BODY_IDLE_TIMEOUT_SECS: u64 = 90; +const BODY_IDLE_TIMEOUT_ERROR_CLASS: &str = "body_idle_timeout"; +const DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS: u64 = 1; + /// Discriminates which byte-stream the client is consuming at any moment. /// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`. /// Lower numeric values correspond to a live or custom stream; `GracePending` @@ -151,7 +155,58 @@ enum DeferredProviderOpenOutcome { enum DeferredProviderOpenState { Pending(Box), - Opening(Pin + Send>>), + Opening(Pin + Send>>), +} + +struct DirectBodyIdleTimeout { + enabled: bool, + deadline: Option, + sleep: Option>>, + last_socket_activity_touch: Option, +} + +impl DirectBodyIdleTimeout { + const fn disabled() -> Self { + Self { enabled: false, deadline: None, sleep: None, last_socket_activity_touch: None } + } + + const fn enabled() -> Self { + Self { enabled: true, deadline: None, sleep: None, last_socket_activity_touch: None } + } + + fn mark_progress(&mut self) -> bool { + if !self.enabled { + return false; + } + + let now = tokio::time::Instant::now(); + self.deadline = Some(now + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)); + self.sleep = None; + + let should_touch_socket = self.last_socket_activity_touch.is_none_or(|last_touch| { + now.duration_since(last_touch) >= tokio::time::Duration::from_secs(DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS) + }); + if should_touch_socket { + self.last_socket_activity_touch = Some(now); + } + should_touch_socket + } + + fn poll_expired(&mut self, cx: &mut Context<'_>) -> bool { + if !self.enabled { + return false; + } + + let deadline = *self.deadline.get_or_insert_with(|| { + tokio::time::Instant::now() + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS) + }); + if tokio::time::Instant::now() >= deadline { + return true; + } + + self.sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline))); + self.sleep.as_mut().is_some_and(|sleep| sleep.as_mut().poll(cx).is_ready()) + } } #[allow(clippy::struct_excessive_bools)] @@ -179,6 +234,7 @@ struct ActiveClientStreamState { custom_video_timeout_secs: u32, custom_video_timeout_mode: Option, custom_video_timeout_sleep: Option>>, + direct_body_idle_timeout: DirectBodyIdleTimeout, /// Set once when the provider stream ends. Read once in Drop. Never queried during streaming. /// Separate from `send_custom_stream_flag` (`StreamMode`). Uses `PROVIDER_END_*` constants. provider_end_reason: AtomicU8, @@ -196,7 +252,7 @@ impl ActiveClientStreamState { CustomVideoStreamType::ProviderConnectionsExhausted => Some(StreamMode::ProviderExhausted), CustomVideoStreamType::LowPriorityPreempted => Some(StreamMode::LowPriorityPreempted), CustomVideoStreamType::Provisioning => Some(StreamMode::Provisioning), - CustomVideoStreamType::UserAccountExpired => None, + CustomVideoStreamType::UserAccountExpired | CustomVideoStreamType::HlsSessionOrLeaseExpired => None, } } @@ -207,13 +263,7 @@ impl ActiveClientStreamState { stream }; if let Some(ctx) = self.timed_stream_context.as_ref() { - TimedClientStream::new( - &ctx.app_state, - stream, - ctx.duration_secs, - self.fingerprint.addr, - ctx.virtual_id, - ) + TimedClientStream::new(&ctx.app_state, stream, ctx.duration_secs, self.fingerprint.addr, ctx.virtual_id) .boxed() } else { stream @@ -247,6 +297,61 @@ impl ActiveClientStreamState { }); } + fn release_stream_and_provider_handle_once(&mut self) { + self.stop_grace_task(); + let addr = self.fingerprint.addr; + let handle = self.provider_handle.take(); + // `provider_handle_released` mirrors `user_stream_released` for the provider slot. + // When preemption already released the handle, `provider_handle` is None and the + // flag is true; sending None is a no-op, but the explicit guard keeps the invariant + // visible and safe against future call-site additions. + let handle_for_cleanup = if self.provider_handle_released { None } else { handle }; + if self.user_stream_released { + if !self.provider_handle_released { + self.provider_handle_released = true; + self.connection_manager + .send_cleanup(CleanupEvent::ReleaseProviderHandle { handle: handle_for_cleanup }); + } + } else { + self.user_stream_released = true; + self.provider_handle_released = true; + self.connection_manager.send_cleanup(CleanupEvent::ReleaseStreamAndProviderHandle { + addr, + stream_uid: self.stream_uid, + handle: handle_for_cleanup, + provider_end_reason: self.provider_end_reason.load(Ordering::Relaxed), + reconnect_count: self.provider_reconnect_count.load(Ordering::Relaxed), + provider_error_class: self.provider_error_class, + provider_http_status: self.provider_http_status, + }); + } + } + + fn stop_direct_body_idle_timeout(&mut self) { + self.provider_stopped = true; + self.preempt_cancelled = None; + self.inner = None; + self.provider_error_class = Some(BODY_IDLE_TIMEOUT_ERROR_CLASS); + self.provider_http_status = None; + let _ = self.provider_end_reason.compare_exchange( + PROVIDER_END_NOT_SET, + PROVIDER_END_ERROR, + Ordering::Relaxed, + Ordering::Relaxed, + ); + warn!( + "Direct body stream idle timeout after {DIRECT_BODY_IDLE_TIMEOUT_SECS}s for {}, terminating stream", + sanitize_sensitive_info(&self.fingerprint.addr.to_string()) + ); + self.release_stream_and_provider_handle_once(); + } + + fn mark_direct_body_progress(&mut self) { + if self.direct_body_idle_timeout.mark_progress() { + self.connection_manager.touch_direct_body_activity(&self.fingerprint.addr); + } + } + fn stop_grace_task(&mut self) { if let Some(task) = self.grace_task_handle.take() { task.abort(); @@ -254,11 +359,7 @@ impl ActiveClientStreamState { } fn clear_finished_grace_task(&mut self) { - if self - .grace_task_handle - .as_ref() - .is_some_and(tokio::task::JoinHandle::is_finished) - { + if self.grace_task_handle.as_ref().is_some_and(tokio::task::JoinHandle::is_finished) { self.grace_task_handle = None; // If the task finished but the flag is still GRACE_PENDING (e.g. the task // panicked or was cancelled before it could update the flag), reset the flag @@ -289,7 +390,8 @@ impl ActiveClientStreamState { flag.store(StreamMode::LowPriorityPreempted as u8, Ordering::Release); } else { // Fallback: create_active_client_stream usually initializes this via stream_grace_period. - self.send_custom_stream_flag = Some(Arc::new(AtomicU8::new(StreamMode::LowPriorityPreempted as u8))); + self.send_custom_stream_flag = + Some(Arc::new(AtomicU8::new(StreamMode::LowPriorityPreempted as u8))); } } else if let Some(flag) = &self.send_custom_stream_flag { flag.store(StreamMode::Inner as u8, Ordering::Release); @@ -381,9 +483,9 @@ impl ActiveClientStreamState { if self.custom_video_timeout_mode != Some(mode) { self.custom_video_timeout_mode = Some(mode); self.custom_video_timeout_sleep = if self.custom_video_timeout_secs > 0 { - Some(Box::pin(tokio::time::sleep(tokio::time::Duration::from_secs( - u64::from(self.custom_video_timeout_secs), - )))) + Some(Box::pin(tokio::time::sleep(tokio::time::Duration::from_secs(u64::from( + self.custom_video_timeout_secs, + ))))) } else { None }; @@ -426,11 +528,7 @@ fn wrap_timed_client_stream_if_needed( None => stream, Some(mins) => { let secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); - if secs > 0 { - TimedClientStream::new(app_state, stream, secs, addr, virtual_id).boxed() - } else { - stream - } + if secs > 0 { TimedClientStream::new(app_state, stream, secs, addr, virtual_id).boxed() } else { stream } } } } @@ -453,8 +551,8 @@ fn create_deferred_provider_open_future( let stream_options = get_stream_options(app_state); let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); let disabled_headers = app_state.get_disabled_headers(); - let mut provider_stream_factory_options = ProviderStreamFactoryOptions::new( - &crate::api::model::ProviderStreamFactoryParams { + let mut provider_stream_factory_options = + ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { addr: fingerprint.addr, item_type: stream_channel.item_type, share_stream: stream_channel.shared, @@ -469,8 +567,7 @@ fn create_deferred_provider_open_future( client_ip: Some(&fingerprint.client_ip), stream_channel: Some(stream_channel), connect_failure_stage: Some(FailureStage::ProviderOpen), - }, - ); + }); provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref())); Some(DeferredProviderOpenState::Pending(Box::new(DeferredProviderOpenContext { @@ -483,11 +580,11 @@ fn create_timed_stream_context(app_state: &Arc, virtual_id: VirtualId) let config = app_state.app_config.config.load(); let mins = config.sleep_timer_mins?; let duration_secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); - (duration_secs > 0).then(|| TimedStreamContext { - app_state: Arc::clone(app_state), - duration_secs, - virtual_id, - }) + (duration_secs > 0).then(|| TimedStreamContext { app_state: Arc::clone(app_state), duration_secs, virtual_id }) +} + +fn should_use_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { + uses_direct_body_idle_timeout(stream_channel) } pub(in crate::api) struct ActiveClientStream { @@ -546,17 +643,24 @@ impl Stream for ActiveClientStream { &client, context.provider_stream_factory_options, ) - .await + .await { - Some((_stream, Some((_headers, _status, _response_url, Some(custom_video_type))))) => { + Some(( + _stream, + Some((_headers, _status, _response_url, Some(custom_video_type))), + )) => { ActiveClientStreamState::mode_for_custom_video_type(custom_video_type) - .map_or(DeferredProviderOpenOutcome::Failed, DeferredProviderOpenOutcome::Mode) + .map_or( + DeferredProviderOpenOutcome::Failed, + DeferredProviderOpenOutcome::Mode, + ) } Some((stream, _stream_info)) => DeferredProviderOpenOutcome::Stream(stream), None => DeferredProviderOpenOutcome::Failed, } }); - self.state.deferred_provider_open = Some(DeferredProviderOpenState::Opening(future)); + self.state.deferred_provider_open = + Some(DeferredProviderOpenState::Opening(future)); continue; } DeferredProviderOpenState::Opening(mut future) => match future.as_mut().poll(cx) { @@ -568,6 +672,7 @@ impl Stream for ActiveClientStream { Poll::Ready(DeferredProviderOpenOutcome::Stream(stream)) => { self.state.provider_reconnect_count.fetch_add(1, Ordering::Relaxed); self.state.inner = Some(self.state.wrap_provider_stream(stream)); + self.state.mark_direct_body_progress(); continue; } Poll::Ready(DeferredProviderOpenOutcome::Mode(mode)) => { @@ -591,7 +696,10 @@ impl Stream for ActiveClientStream { } match self.state.inner.as_mut().map(|inner| Pin::new(inner).poll_next(cx)) { - Some(Poll::Ready(Some(Ok(bytes)))) => return Poll::Ready(Some(Ok(bytes))), + Some(Poll::Ready(Some(Ok(bytes)))) => { + self.state.mark_direct_body_progress(); + return Poll::Ready(Some(Ok(bytes))); + } Some(Poll::Ready(Some(Err(e)))) => { error!("Inner stream error: {e:?}"); self.state.provider_error_class = Some(e.provider_error_class()); @@ -623,7 +731,13 @@ impl Stream for ActiveClientStream { return Poll::Pending; } - Some(Poll::Pending) => return Poll::Pending, + Some(Poll::Pending) => { + if self.state.direct_body_idle_timeout.poll_expired(cx) { + self.state.stop_direct_body_idle_timeout(); + return Poll::Ready(None); + } + return Poll::Pending; + } } } @@ -684,32 +798,7 @@ impl Stream for ActiveClientStream { impl Drop for ActiveClientStream { fn drop(&mut self) { - self.state.stop_grace_task(); - let addr = self.state.fingerprint.addr; - let handle = self.state.provider_handle.take(); - // `provider_handle_released` mirrors `user_stream_released` for the provider slot. - // When preemption already released the handle, `provider_handle` is None and the - // flag is true — sending None here would be a no-op, but the explicit guard makes - // the invariant visible and safe against future call-site additions. - let handle_for_cleanup = if self.state.provider_handle_released { None } else { handle }; - if self.state.user_stream_released { - if !self.state.provider_handle_released { - self.state.provider_handle_released = true; - self.state.connection_manager.send_cleanup(CleanupEvent::ReleaseProviderHandle { handle: handle_for_cleanup }); - } - } else { - self.state.user_stream_released = true; - self.state.provider_handle_released = true; - self.state.connection_manager.send_cleanup(CleanupEvent::ReleaseStreamAndProviderHandle { - addr, - stream_uid: self.state.stream_uid, - handle: handle_for_cleanup, - provider_end_reason: self.state.provider_end_reason.load(Ordering::Relaxed), - reconnect_count: self.state.provider_reconnect_count.load(Ordering::Relaxed), - provider_error_class: self.state.provider_error_class, - provider_http_status: self.state.provider_http_status, - }); - } + self.state.release_stream_and_provider_handle_once(); } } @@ -740,6 +829,11 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam let virtual_id = stream_channel.virtual_id; let is_shared_source_stream = stream_channel.shared && stream_details.stream.is_some(); + let direct_body_idle_timeout = if should_use_direct_body_idle_timeout(&stream_channel) { + DirectBodyIdleTimeout::enabled() + } else { + DirectBodyIdleTimeout::disabled() + }; let registered_stream = app_state .connection_manager .update_connection(crate::api::model::ConnectionParams { @@ -810,16 +904,12 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam // Compute deferred provider open before moving stream_details into the grace task. let deferred_provider_open = create_deferred_provider_open_future(app_state, &stream_details, fingerprint, &stream_channel, req_headers); - let timed_stream_context = deferred_provider_open - .as_ref() - .and_then(|_| create_timed_stream_context(app_state, virtual_id)); + let timed_stream_context = + deferred_provider_open.as_ref().and_then(|_| create_timed_stream_context(app_state, virtual_id)); let stream_taken = stream_details.stream.take(); let has_deferred_open = stream_details.has_deferred_provider_open(); - let grace_waker = if grant_user_grace_period || stream_details.provider_grace_active { - Some(Arc::clone(&waker)) - } else { - None - }; + let grace_waker = + if grant_user_grace_period || stream_details.provider_grace_active { Some(Arc::clone(&waker)) } else { None }; let (grace_stop_flag, grace_task_handle) = stream_grace_period(GracePeriodParams { app_state: Arc::clone(app_state), stream_details, @@ -883,9 +973,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam .map(|token| Box::pin(token.clone().cancelled_owned())); let mut send_custom_stream_flag = grace_stop_flag; - if send_custom_stream_flag.is_none() - && preempt_cancelled.is_some() - && custom_video.low_priority_preempted.is_some() + if send_custom_stream_flag.is_none() && preempt_cancelled.is_some() && custom_video.low_priority_preempted.is_some() { send_custom_stream_flag = Some(Arc::new(AtomicU8::new(StreamMode::Inner as u8))); } @@ -912,6 +1000,7 @@ pub(crate) async fn create_active_client_stream(request: ActiveClientStreamParam custom_video_timeout_secs, custom_video_timeout_mode: None, custom_video_timeout_sleep: None, + direct_body_idle_timeout, provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), provider_error_class: None, provider_http_status: None, @@ -982,9 +1071,8 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op }; if provider_grace_check.is_some() || user_grace_check.is_some() { - let stream_strategy_flag = Arc::new(AtomicU8::new( - if hold_stream { StreamMode::GracePending as u8 } else { StreamMode::Inner as u8 }, - )); + let stream_strategy_flag = + Arc::new(AtomicU8::new(if hold_stream { StreamMode::GracePending as u8 } else { StreamMode::Inner as u8 })); let stream_strategy_flag_copy = Arc::clone(&stream_strategy_flag); let grace_period_millis = grace_period.period_millis; @@ -1001,8 +1089,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op // out of GRACE_PENDING so the client stream is not hung indefinitely. // Allow grace_period_millis for the intentional delay plus a 10-second buffer // for the async connection checks that follow. - let grace_task_timeout = - tokio::time::Duration::from_millis(grace_period_millis.saturating_add(10_000)); + let grace_task_timeout = tokio::time::Duration::from_millis(grace_period_millis.saturating_add(10_000)); // Clone handles for use in the timeout fallback, in case the inner async block is cancelled. let flag_for_fallback = Arc::clone(&stream_strategy_flag_copy); let waker_for_fallback = waker.clone(); @@ -1011,8 +1098,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op let pending_username_timeout = pending_username.clone(); let grace_task_handle = tokio::spawn(async move { let timed_out = tokio::time::timeout(grace_task_timeout, async move { - let deadline = - tokio::time::Instant::now() + tokio::time::Duration::from_millis(grace_period_millis); + let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(grace_period_millis); let mut pending_wake_source = PendingProviderWakeSource::Activated; loop { let capacity_wait = capacity_notify.notified(); @@ -1025,9 +1111,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op None => true, }; let provider_ok = match &provider_grace_check { - Some(provider_name) => { - !provider_manager.is_over_limit(provider_name).await - } + Some(provider_name) => !provider_manager.is_over_limit(provider_name).await, None => true, }; if user_ok && provider_ok { @@ -1073,7 +1157,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op ctx, grace_kind, ) - .await; + .await; match remaining_result.admission.permission { shared::model::UserConnectionPermission::Allowed | shared::model::UserConnectionPermission::GracePeriod => { @@ -1083,10 +1167,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op } shared::model::UserConnectionPermission::Exhausted => { // Remaining strategies exhausted — final UserExhausted. - stream_strategy_flag_copy.store( - StreamMode::UserExhausted as u8, - Ordering::Release, - ); + stream_strategy_flag_copy.store(StreamMode::UserExhausted as u8, Ordering::Release); connection_manager .update_stream_detail( &fingerprint.addr, @@ -1103,12 +1184,12 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op } } else { // No grace context — immediate UserExhausted. - stream_strategy_flag_copy.store( - StreamMode::UserExhausted as u8, - Ordering::Release, - ); + stream_strategy_flag_copy.store(StreamMode::UserExhausted as u8, Ordering::Release); connection_manager - .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::UserConnectionsExhausted) + .update_stream_detail( + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) .await; connection_manager.shared_stream_manager.release_connection(&fingerprint.addr, true).await; info!("User connections exhausted for active clients: {username}"); @@ -1129,13 +1210,18 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op "Provider grace period exhausted; provisioning for active clients: {provider_name}" ); let app_state = Arc::clone(&app_state); - let input = (*provisioning_info.input).clone(); + let input_name = Arc::clone(&provisioning_info.input.name); let stop_signal = provisioning_info.stop_signal; let addr = fingerprint.addr; tokio::spawn(async move { - if let Err(err) = - run_panel_api_provisioning_probe(app_state, input, stop_signal, addr, virtual_id) - .await + if let Err(err) = run_panel_api_provisioning_probe( + app_state, + input_name, + stop_signal, + addr, + virtual_id, + ) + .await { error!("Error running Probe: {err:?}"); } @@ -1149,7 +1235,10 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op ) .await; // Release the shared stream subscription to stop the subscriber loop - connection_manager.shared_stream_manager.release_connection(&fingerprint.addr, true).await; + connection_manager + .shared_stream_manager + .release_connection(&fingerprint.addr, true) + .await; info!("Provider connections exhausted for active clients: {provider_name}"); } updated = true; @@ -1165,9 +1254,8 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op // PendingProvider (Hold): activate on success, expire on failure. if hold_stream { if let (Some(token), Some(version)) = (session_token.as_deref(), pending_provider_version) { - let _transition_guard = active_users - .acquire_playback_transition(&pending_username, token) - .await; + let _transition_guard = + active_users.acquire_playback_transition(&pending_username, token).await; if updated { active_users .expire_pending_provider(&pending_username, token, version, pending_wake_source) @@ -1183,17 +1271,11 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op // GraceActive (Instant): activate on success, expire on failure. // grace_active_version is Some when the session is in `GraceActive` lifecycle. if let (Some(token), Some(version)) = (session_token.as_deref(), grace_active_version) { - let _transition_guard = active_users - .acquire_playback_transition(&pending_username, token) - .await; + let _transition_guard = active_users.acquire_playback_transition(&pending_username, token).await; if updated { - active_users - .expire_grace_active(&pending_username, token, version) - .await; + active_users.expire_grace_active(&pending_username, token, version).await; } else { - active_users - .activate_grace_active(&pending_username, token, version) - .await; + active_users.activate_grace_active(&pending_username, token, version).await; } } @@ -1207,7 +1289,7 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op w.wake(); } }) - .await; + .await; if timed_out.is_err() { // Grace task exceeded its budget without updating the flag — reset GRACE_PENDING @@ -1216,20 +1298,21 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op // a PendingProvider / GraceActive session that was never resolved would cause // the next admission attempt to incorrectly skip re-evaluation. if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), pending_provider_version) { - let _transition_guard = active_users_timeout - .acquire_playback_transition(&pending_username_timeout, token) - .await; + let _transition_guard = + active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; active_users_timeout - .expire_pending_provider(&pending_username_timeout, token, version, PendingProviderWakeSource::Timeout) + .expire_pending_provider( + &pending_username_timeout, + token, + version, + PendingProviderWakeSource::Timeout, + ) .await; } if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), grace_active_version) { - let _transition_guard = active_users_timeout - .acquire_playback_transition(&pending_username_timeout, token) - .await; - active_users_timeout - .expire_grace_active(&pending_username_timeout, token, version) - .await; + let _transition_guard = + active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; + active_users_timeout.expire_grace_active(&pending_username_timeout, token, version).await; } error!("Grace period task timed out; resetting stream flag to prevent client hang"); let _ = flag_for_fallback.compare_exchange( @@ -1251,37 +1334,43 @@ fn stream_grace_period(request: GracePeriodParams) -> (Option>, Op #[cfg(test)] mod tests { use super::{ - create_active_client_stream, stream_grace_period, ActiveClientStream, ActiveClientStreamParams, - ActiveClientStreamState, - CustomVideoBuffers, DeferredProviderOpenOutcome, DeferredProviderOpenState, GracePeriodParams, StreamMode, - TimedStreamContext, + ActiveClientStream, ActiveClientStreamParams, ActiveClientStreamState, CustomVideoBuffers, + DIRECT_BODY_IDLE_TIMEOUT_SECS, DeferredProviderOpenOutcome, DeferredProviderOpenState, DirectBodyIdleTimeout, + GracePeriodParams, StreamMode, TimedStreamContext, create_active_client_stream, + should_use_direct_body_idle_timeout, stream_grace_period, }; use crate::api::api_utils::GraceResolutionContext; use crate::api::model::connection_manager::PROVIDER_END_NOT_SET; use crate::{ api::model::{ - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, CreateUserSessionParams, - CustomVideoStreamType, DownloadQueue, EventManager, MetadataUpdateManager, PlaylistStorageState, - SharedStreamManager, StreamDetails, StreamError, UpdateGuard, + ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, + CreateUserSessionParams, CustomVideoStreamType, DownloadQueue, EventManager, MetadataUpdateManager, + PlaylistStorageState, SharedStreamManager, StreamDetails, StreamError, UpdateGuard, }, auth::Fingerprint, - model::{AppConfig, Config, ConfigInput, GracePeriodOptions, MediaToolCapabilities, ProcessTargets, ProxyUserCredentials, SourcesConfig, StreamConfig}, + model::{ + AppConfig, Config, ConfigInput, GracePeriodOptions, MediaToolCapabilities, ProcessTargets, + ProxyUserCredentials, SourcesConfig, StreamConfig, + }, utils::{FileLockManager, GeoIp}, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::http::HeaderMap; use bytes::Bytes; - use futures::{pin_mut, StreamExt}; + use futures::{StreamExt, pin_mut}; use reqwest::Client; use shared::{ - model::{AdmissionStrategy, ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, UserConnectionPermission, XtreamCluster}, + model::{ + AdmissionStrategy, ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, + UserConnectionPermission, XtreamCluster, + }, utils::Internable, }; use std::{ collections::HashMap, sync::{ - atomic::{AtomicU8, Ordering}, Arc, + atomic::{AtomicU8, Ordering}, }, time::Duration, }; @@ -1342,13 +1431,7 @@ mod tests { let config = app_cfg.config.load(); let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); - Arc::new(ConnectionManager::new( - &user_manager, - &provider_manager, - &shared_manager, - &event_manager, - None, - )) + Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)) } fn create_test_app_state() -> Arc { @@ -1386,6 +1469,8 @@ mod tests { downloads: Arc::new(DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(crate::api::model::HlsProxyManager::new()), + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), active_users, active_provider, connection_manager, @@ -1412,6 +1497,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }), user_access_control: true, ..Config::default() @@ -1453,6 +1539,8 @@ mod tests { downloads: Arc::new(DownloadQueue::new()), cache: Arc::new(ArcSwapOption::default()), shared_stream_manager, + hls_proxy: Arc::new(crate::api::model::HlsProxyManager::new()), + hls_provisioning: Arc::new(crate::api::model::HlsProvisioningState::new()), active_users, active_provider, connection_manager, @@ -1503,6 +1591,15 @@ mod tests { channel } + fn create_test_video_stream_channel(virtual_id: u32, url: &str) -> StreamChannel { + let mut channel = create_test_stream_channel(virtual_id, url); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.group = "Movies".intern(); + channel.url = url.into(); + channel + } + fn create_deferred_provider_grace_details( provider_name: &Arc, provider_handle: crate::api::model::ProviderHandle, @@ -1531,11 +1628,7 @@ mod tests { provider_name: &Arc, deferred_addr: std::net::SocketAddr, session_token: Option<&str>, - ) -> ( - Arc, - tokio::task::JoinHandle<()>, - crate::api::model::ProviderHandle, - ) { + ) -> (Arc, tokio::task::JoinHandle<()>, crate::api::model::ProviderHandle) { let deferred_handle = app_state .active_provider .acquire_exact_connection_with_grace( @@ -1552,10 +1645,7 @@ mod tests { let test_user = create_test_user("grace-user"); let test_fingerprint = create_test_fingerprint(deferred_addr); let pending_provider_version = if let Some(token) = session_token { - app_state - .active_users - .pending_provider_version(&test_user.username, token) - .await + app_state.active_users.pending_provider_version(&test_user.username, token).await } else { None }; @@ -1608,11 +1698,7 @@ mod tests { event_manager: Arc::new(EventManager::new()), waker: None, connection_manager, - fingerprint: Arc::new(Fingerprint::new( - "fp-key".to_string(), - "127.0.0.1".to_string(), - addr, - )), + fingerprint: Arc::new(Fingerprint::new("fp-key".to_string(), "127.0.0.1".to_string(), addr)), stream_uid: None, provider_stopped: true, user_stream_released: true, @@ -1620,6 +1706,7 @@ mod tests { custom_video_timeout_secs: 5, custom_video_timeout_mode: None, custom_video_timeout_sleep: None, + direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), provider_error_class: None, provider_http_status: None, @@ -1657,6 +1744,27 @@ mod tests { )); } + #[test] + fn test_direct_body_idle_timeout_only_applies_to_direct_vod_series_streams() { + let video = create_test_video_stream_channel(1, "http://provider-1.example/movie/1.mkv"); + assert!(should_use_direct_body_idle_timeout(&video)); + + let mut series = create_test_video_stream_channel(2, "http://provider-1.example/series/2.mkv"); + series.item_type = PlaylistItemType::Series; + series.cluster = XtreamCluster::Series; + assert!(should_use_direct_body_idle_timeout(&series)); + + let live = create_test_stream_channel(3, "http://provider-1.example/live/3.ts"); + assert!(!should_use_direct_body_idle_timeout(&live)); + + let shared_video = { + let mut channel = video.clone(); + channel.shared = true; + channel + }; + assert!(!should_use_direct_body_idle_timeout(&shared_video)); + } + #[tokio::test] async fn test_provisioning_without_custom_video_terminates_immediately_with_timeout_configured() { assert_missing_custom_video_terminates(StreamMode::Provisioning, true).await; @@ -1769,7 +1877,8 @@ mod tests { .await .expect("holder should consume the provider's live capacity"); let (_flag, grace_task, deferred_handle) = - start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace")).await; + start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace")) + .await; app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; @@ -1780,7 +1889,10 @@ mod tests { .get_and_update_user_session(&user.username, "tok-grace") .await .expect("session should still exist"); - assert!(!matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), "capacity notify should clear pending provider state"); + assert!( + !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), + "capacity notify should clear pending provider state" + ); assert_eq!(session.permission, UserConnectionPermission::Allowed); app_state.connection_manager.release_provider_handle(Some(deferred_handle)).await; @@ -1835,7 +1947,7 @@ mod tests { deferred_addr, Some("tok-grace-stale"), ) - .await; + .await; let replacement_version = app_state .active_users @@ -1857,9 +1969,7 @@ mod tests { .get_and_update_user_session(&user.username, "tok-grace-stale") .await .expect("session should still exist"); - let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = - &session.lifecycle - else { + let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { panic!("stale grace task must not clear the replacement pending provider state") }; assert_eq!(pending.version, replacement_version); @@ -1916,7 +2026,7 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await; pin_mut!(stream); assert!( @@ -1991,7 +2101,7 @@ mod tests { meter_uid: 0, meter_stream: false, }) - .await; + .await; pin_mut!(stream); assert!( @@ -2031,11 +2141,7 @@ mod tests { deferred_provider_open: Some(DeferredProviderOpenState::Opening(Box::pin(async { DeferredProviderOpenOutcome::Stream(futures::stream::pending::>().boxed()) }))), - timed_stream_context: Some(TimedStreamContext { - app_state, - duration_secs: 1, - virtual_id: 1, - }), + timed_stream_context: Some(TimedStreamContext { app_state, duration_secs: 1, virtual_id: 1 }), preempt_cancelled: None, grace_task_handle: None, provisionable: false, @@ -2050,11 +2156,7 @@ mod tests { event_manager: Arc::new(EventManager::new()), waker: None, connection_manager, - fingerprint: Arc::new(Fingerprint::new( - "fp-timeout".to_string(), - "127.0.0.1".to_string(), - addr, - )), + fingerprint: Arc::new(Fingerprint::new("fp-timeout".to_string(), "127.0.0.1".to_string(), addr)), stream_uid: None, provider_stopped: false, user_stream_released: true, @@ -2062,6 +2164,7 @@ mod tests { custom_video_timeout_secs: 0, custom_video_timeout_mode: None, custom_video_timeout_sleep: None, + direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), provider_error_class: None, provider_http_status: None, @@ -2078,10 +2181,7 @@ mod tests { tokio::time::advance(Duration::from_secs(2)).await; let result = tokio::time::timeout(Duration::from_millis(1), stream.next()).await; - assert!( - result.is_ok(), - "deferred-open stream should stop once the configured sleep timer expires" - ); + assert!(result.is_ok(), "deferred-open stream should stop once the configured sleep timer expires"); match result { Ok(joined) => assert!( joined.is_none(), @@ -2134,8 +2234,13 @@ mod tests { ) .await .expect("holder should consume the provider's live capacity"); - let (flag, grace_task, deferred_handle) = - start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace-timeout")).await; + let (flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution( + &app_state, + &provider_name, + deferred_addr, + Some("tok-grace-timeout"), + ) + .await; assert_eq!( StreamMode::from_u8(flag.load(Ordering::Acquire)), @@ -2161,7 +2266,10 @@ mod tests { .get_and_update_user_session(&user.username, "tok-grace-timeout") .await .expect("session should still exist"); - assert!(!matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), "timeout expiry should clear pending provider state"); + assert!( + !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), + "timeout expiry should clear pending provider state" + ); assert_eq!(session.permission, UserConnectionPermission::Exhausted); app_state.connection_manager.release_provider_handle(Some(holder_handle)).await; @@ -2473,7 +2581,7 @@ mod tests { meter_uid: 55, meter_stream: true, }) - .await; + .await; pin_mut!(stream); let first_chunk = stream.next().await; @@ -2495,4 +2603,56 @@ mod tests { assert_eq!(entries[0].rate_kbps, 1); assert_eq!(entries[0].total_kb, 3); } + + #[tokio::test(start_paused = true)] + async fn test_direct_vod_body_idle_timeout_releases_active_stream() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55031".parse().unwrap_or_else(|_| unreachable!()); + let test_user = create_test_user("vod-user"); + let test_fingerprint = create_test_fingerprint(addr); + let provider_stream = + futures::stream::once(async { Ok(Bytes::from_static(b"vod")) }).chain(futures::stream::pending()).boxed(); + let stream_details = StreamDetails::from_stream(provider_stream, GracePeriodOptions::default()); + + let stream = create_active_client_stream(ActiveClientStreamParams { + stream_details, + app_state: &app_state, + user: &test_user, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + fingerprint: &test_fingerprint, + stream_channel: create_test_video_stream_channel(1, "http://provider-1.example/movie/1.mkv"), + socket_bound: false, + session_token: None, + req_headers: &HeaderMap::default(), + meter_uid: 0, + meter_stream: false, + }) + .await; + pin_mut!(stream); + + let first_chunk = stream.next().await; + assert!(matches!(first_chunk, Some(Ok(ref bytes)) if bytes.as_ref() == b"vod")); + assert_eq!(app_state.active_users.active_streams().await.len(), 1); + assert!( + matches!(futures::poll!(stream.next()), std::task::Poll::Pending), + "pending VOD body should wait until the direct body idle timeout elapses" + ); + + tokio::time::advance(Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)).await; + tokio::task::yield_now().await; + + assert!(stream.next().await.is_none(), "VOD body idle timeout should terminate the stream"); + + for _ in 0..20 { + if app_state.active_users.active_streams().await.is_empty() { + return; + } + tokio::task::yield_now().await; + } + assert!( + app_state.active_users.active_streams().await.is_empty(), + "cleanup worker should remove the timed-out VOD stream from active streams" + ); + } } diff --git a/backend/src/api/model/streams/provider_stream.rs b/backend/src/api/model/streams/provider_stream.rs index d2b863d8f..d438ff817 100644 --- a/backend/src/api/model/streams/provider_stream.rs +++ b/backend/src/api/model/streams/provider_stream.rs @@ -13,10 +13,10 @@ use axum::response::IntoResponse; use log::trace; use reqwest::StatusCode; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use shared::error::TuliproxError; use shared::model::PlaylistItemType; use std::{fmt, net::SocketAddr, str::FromStr, sync::Arc}; use tokio_util::sync::CancellationToken; -use shared::error::TuliproxError; #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] pub enum CustomVideoStreamType { @@ -26,6 +26,7 @@ pub enum CustomVideoStreamType { LowPriorityPreempted, UserAccountExpired, Provisioning, + HlsSessionOrLeaseExpired, } impl fmt::Display for CustomVideoStreamType { @@ -37,6 +38,7 @@ impl fmt::Display for CustomVideoStreamType { CustomVideoStreamType::LowPriorityPreempted => "low_priority_preempted", CustomVideoStreamType::UserAccountExpired => "user_account_expired", CustomVideoStreamType::Provisioning => "provisioning", + CustomVideoStreamType::HlsSessionOrLeaseExpired => "hls_session_or_lease_expired", }; write!(f, "{s}") } @@ -53,6 +55,7 @@ impl FromStr for CustomVideoStreamType { "low_priority_preempted" => Ok(Self::LowPriorityPreempted), "user_account_expired" => Ok(Self::UserAccountExpired), "provisioning" => Ok(Self::Provisioning), + "hls_session_or_lease_expired" => Ok(Self::HlsSessionOrLeaseExpired), _ => Err(TuliproxError::Config(format!("Unknown stream type: {s}"))), } } @@ -106,10 +109,10 @@ fn apply_custom_stream_timeout(cfg: &AppConfig, stream: BoxedProviderStream) -> } /// Returns the value of `custom_stream_response_enabled` from the main config. -/// When `false`, the 6 custom-video factories (`channel_unavailable`, +/// When `false`, the custom-video factories (`channel_unavailable`, /// `user_connections_exhausted`, `provider_connections_exhausted`, -/// `low_priority_preempted`, `user_account_expired`, `panel_api_provisioning`) skip -/// the configured MPEG-TS video and the call sites return +/// `low_priority_preempted`, `user_account_expired`, `panel_api_provisioning`, +/// `hls_session_or_lease_expired`) skip the configured MPEG-TS video and the call sites return /// `custom_stream_response_error_status` instead. This allows a downstream Nginx /// with `proxy_intercept_errors on;` to sever the socket instead of seeing an /// infinite 200 OK loop. @@ -233,6 +236,13 @@ ok_custom_stream_factory!( "Streaming response panel api provisioning" ); +ok_custom_stream_factory!( + create_hls_session_or_lease_expired_stream, + hls_session_or_lease_expired, + CustomVideoStreamType::HlsSessionOrLeaseExpired, + "Streaming response hls session or lease expired" +); + pub fn create_panel_api_provisioning_stream_with_stop( cfg: &AppConfig, headers: &[(String, String)], @@ -264,7 +274,7 @@ pub fn create_custom_video_stream_response( let config = &app_state.app_config; if let (Some(stream), Some((headers, status_code, _, _))) = match video_response { CustomVideoStreamType::ChannelUnavailable => { - create_channel_unavailable_stream(config, &[], StatusCode::BAD_REQUEST) + create_channel_unavailable_stream(config, &[], StatusCode::OK) } CustomVideoStreamType::UserConnectionsExhausted => create_user_connections_exhausted_stream(config, &[]), CustomVideoStreamType::ProviderConnectionsExhausted => { @@ -273,6 +283,7 @@ pub fn create_custom_video_stream_response( CustomVideoStreamType::LowPriorityPreempted => create_low_priority_preempted_stream(config, &[]), CustomVideoStreamType::UserAccountExpired => create_user_account_expired_stream(config, &[]), CustomVideoStreamType::Provisioning => create_panel_api_provisioning_stream(config, &[]), + CustomVideoStreamType::HlsSessionOrLeaseExpired => create_hls_session_or_lease_expired_stream(config, &[]), } { app_state.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProviderConnection { addr: *addr, @@ -373,6 +384,8 @@ mod tests { low_priority_preempted: None, user_account_expired: None, panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), }))); app_cfg } @@ -385,7 +398,14 @@ mod tests { } #[test] - fn test_channel_unavailable_preserves_supplied_status_code() { + fn test_hls_session_or_lease_expired_custom_video_type_roundtrip() { + let parsed = CustomVideoStreamType::from_str("hls_session_or_lease_expired") + .expect("hls_session_or_lease_expired should parse as custom video type"); + assert_eq!(parsed.to_string(), "hls_session_or_lease_expired"); + } + + #[tokio::test] + async fn test_channel_unavailable_preserves_supplied_status_code() { let app_cfg = create_test_app_config_with_channel_unavailable(); let (_stream, info) = create_channel_unavailable_stream(&app_cfg, &[], StatusCode::SERVICE_UNAVAILABLE); @@ -399,8 +419,8 @@ mod tests { /// each (field, stream type, description) tuple. Pick one representative /// factory — `create_user_connections_exhausted_stream` — and verify the /// macro forwards the correct `CustomVideoStreamType` to the stream info. - #[test] - fn test_ok_custom_stream_factory_macro_forwards_video_type() { + #[tokio::test] + async fn test_ok_custom_stream_factory_macro_forwards_video_type() { use super::create_user_connections_exhausted_stream; use crate::api::model::TransportStreamBuffer; @@ -476,7 +496,8 @@ mod tests { /// The centralisation in `create_video_stream` must apply to the macro-generated /// factories as well (`user_connections_exhausted`, `provider_connections_exhausted`, - /// `low_priority_preempted`, `user_account_expired`, `panel_api_provisioning`). + /// `low_priority_preempted`, `user_account_expired`, `panel_api_provisioning`, + /// `hls_session_or_lease_expired`). #[test] fn test_hiding_custom_video_streams_applies_to_macro_factory() { use super::create_user_connections_exhausted_stream; @@ -498,8 +519,8 @@ mod tests { /// Regression guard: when `custom_stream_response_enabled` is true (the default), the /// factory must still return the configured video with the supplied status code. - #[test] - fn test_displaying_custom_video_streams_serves_video_with_supplied_status() { + #[tokio::test] + async fn test_displaying_custom_video_streams_serves_video_with_supplied_status() { let app_cfg = create_test_app_config_with_channel_unavailable(); let (stream, info) = create_channel_unavailable_stream(&app_cfg, &[], StatusCode::SERVICE_UNAVAILABLE); assert!(stream.is_some(), "stream must be produced when custom_stream_response_enabled is true"); diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/src/api/model/streams/provider_stream_factory.rs index c90919c22..1d7fb7405 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/src/api/model/streams/provider_stream_factory.rs @@ -35,9 +35,10 @@ use std::{ }; use tokio_util::sync::CancellationToken; use url::Url; -use shared::model::ConnectFailureReason; -use shared::utils::{Internable, DEFAULT_USER_AGENT}; -use shared::model::{FailureStage}; +use shared::{ + model::{ConnectFailureReason, FailureStage}, + utils::Internable, + defaults::DEFAULT_USER_AGENT}; const RETRY_SECONDS: u64 = 5; const ERR_MAX_RETRY_COUNT: u32 = 5; @@ -861,7 +862,7 @@ pub async fn create_provider_stream( if let (Some(boxed_provider_stream), response_info) = create_channel_unavailable_stream( &app_state.app_config, &get_response_headers(stream_options.get_headers()), - status, + StatusCode::OK, ) { return Some((boxed_provider_stream, response_info)); } diff --git a/backend/src/api/model/streams/shared_stream_manager.rs b/backend/src/api/model/streams/shared_stream_manager.rs index 2cb3dafdd..2fca85aca 100644 --- a/backend/src/api/model/streams/shared_stream_manager.rs +++ b/backend/src/api/model/streams/shared_stream_manager.rs @@ -341,7 +341,7 @@ impl SharedStreamState { let burst_buffer = Arc::clone(&self.burst_buffer); let burst_buffer_for_log = Arc::clone(&self.burst_buffer); let live_notification = Arc::clone(&self.live_notification); - let timeout_duration = Duration::from_secs(300); + let timeout_duration = Duration::from_mins(5); let idle_check_interval = Duration::from_secs(1); let mut last_active = Instant::now(); let mut last_lag_log = Instant::now().checked_sub(Duration::from_secs(10)).unwrap_or_else(Instant::now); diff --git a/backend/src/api/model/streams/throttled_stream.rs b/backend/src/api/model/streams/throttled_stream.rs index f9f8cb480..f89823693 100644 --- a/backend/src/api/model/streams/throttled_stream.rs +++ b/backend/src/api/model/streams/throttled_stream.rs @@ -7,12 +7,13 @@ use std::{ task::{Context, Poll}, time::Duration, }; -use tokio::time::{sleep, Sleep}; +use tokio::time::{sleep, Instant, Sleep}; pub struct ThrottledStream { inner: S, rate_bytes_per_sec: f64, - next_delay: Option>>, + delay: Pin>, + delay_active: bool, } impl ThrottledStream { @@ -20,7 +21,12 @@ impl ThrottledStream { pub fn new(inner: S, throttle_kbps: usize) -> Self { assert!(throttle_kbps > 0, "Rate must be greater than 0"); let rate_bytes_per_sec = (throttle_kbps as f64) * 1000.0 / 8.0; - Self { inner, rate_bytes_per_sec, next_delay: None } + Self { + inner, + rate_bytes_per_sec, + delay: Box::pin(sleep(Duration::ZERO)), + delay_active: false, + } } } @@ -34,17 +40,10 @@ where fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = &mut *self; - // Check if there's an active delay - if let Some(mut delay) = this.next_delay.take() { - match delay.as_mut().poll(cx) { - Poll::Ready(()) => { - // Delay completed, proceed to poll inner stream - } - Poll::Pending => { - // Re-insert the delay and return Pending - this.next_delay = Some(delay); - return Poll::Pending; - } + if this.delay_active { + match this.delay.as_mut().poll(cx) { + Poll::Ready(()) => this.delay_active = false, + Poll::Pending => return Poll::Pending, } } @@ -55,8 +54,8 @@ where let delay_secs = (len / this.rate_bytes_per_sec).max(0.001); let delay_duration = Duration::from_secs_f64(delay_secs); - // Schedule the next delay - this.next_delay = Some(Box::pin(sleep(delay_duration))); + this.delay.as_mut().reset(Instant::now() + delay_duration); + this.delay_active = true; Poll::Ready(Some(Ok(bytes))) } @@ -71,3 +70,32 @@ where } impl Unpin for ThrottledStream {} + +#[cfg(test)] +mod tests { + use super::ThrottledStream; + use crate::api::model::StreamError; + use bytes::Bytes; + use futures::{stream, StreamExt}; + use std::{task::Poll, time::Duration}; + + #[tokio::test(start_paused = true)] + async fn throttled_stream_delays_between_chunks_without_recreating_stream_state() { + let inner = stream::iter([ + Ok::(Bytes::from_static(b"abcd")), + Ok::(Bytes::from_static(b"efgh")), + ]); + let mut stream = ThrottledStream::new(inner, 32); + + assert_eq!(stream.next().await.unwrap().unwrap(), Bytes::from_static(b"abcd")); + assert!( + matches!(futures::poll!(stream.next()), Poll::Pending), + "second chunk must wait for the throttle delay" + ); + + tokio::time::advance(Duration::from_millis(1)).await; + tokio::task::yield_now().await; + + assert_eq!(stream.next().await.unwrap().unwrap(), Bytes::from_static(b"efgh")); + } +} diff --git a/backend/src/api/model/streams/timed_client_stream.rs b/backend/src/api/model/streams/timed_client_stream.rs index 91dbc2040..9e322dfb1 100644 --- a/backend/src/api/model/streams/timed_client_stream.rs +++ b/backend/src/api/model/streams/timed_client_stream.rs @@ -6,7 +6,8 @@ use bytes::Bytes; use futures::Stream; use shared::{ model::VirtualId, - utils::{default_kick_secs, sanitize_sensitive_info}, + utils::{sanitize_sensitive_info}, + defaults::{default_kick_secs}, }; use std::{ future::Future, diff --git a/backend/src/api/model/streams/transport_stream_buffer.rs b/backend/src/api/model/streams/transport_stream_buffer.rs index b30b5835c..6466a1b43 100644 --- a/backend/src/api/model/streams/transport_stream_buffer.rs +++ b/backend/src/api/model/streams/transport_stream_buffer.rs @@ -331,7 +331,11 @@ fn calculate_pcr_duration_ticks(buffer: &[u8]) -> Option { type PacketIndices = Vec<(usize, Option<(usize, Option, u16)>)>; pub struct TransportStreamBuffer { - buffer: Arc>, + // `Bytes` instead of `Arc>`: cloning the inner payload (which + // happens once per HLS-CVS fallback response) is then a refcount bump + // instead of a deep copy. All existing `&self.buffer[..]` / `.len()` / + // `.is_empty()` call sites continue to work via `Bytes`'s `Deref`. + buffer: Bytes, packet_indices: Arc, current_pos: usize, current_dts: u64, @@ -363,7 +367,7 @@ impl std::fmt::Debug for TransportStreamBuffer { impl Clone for TransportStreamBuffer { fn clone(&self) -> Self { Self { - buffer: Arc::clone(&self.buffer), + buffer: self.buffer.clone(), packet_indices: Arc::clone(&self.packet_indices), current_pos: 0, current_dts: 0, @@ -421,7 +425,7 @@ impl TransportStreamBuffer { } Self { - buffer: Arc::new(raw), + buffer: Bytes::from(raw), current_pos: 0, current_dts: 0, timestamp_offset: 0, @@ -449,6 +453,16 @@ impl TransportStreamBuffer { } } + pub fn as_bytes(&self) -> &[u8] { &self.buffer } + + /// Cheap clone of the underlying buffer as `Bytes` (refcount bump). + /// Use this from response builders to avoid `Bytes::copy_from_slice(&[u8])`. + pub fn clone_bytes(&self) -> Bytes { self.buffer.clone() } + + pub fn duration_ms(&self) -> Option { + (self.stream_duration_90khz > 0).then(|| self.stream_duration_90khz.saturating_mul(1_000) / 90_000) + } + #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] pub fn estimated_bitrate_kbps(&self) -> Option { if self.buffer.is_empty() || self.stream_duration_90khz == 0 { diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index 34ecbca9b..4fd94f420 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -7,18 +7,19 @@ use crate::{ }, }, model::{ - is_input_expired, ConfigInput, ConfigInputAlias, GracePeriodOptions, PanelApiConfig, PanelApiQueryParam, + is_input_expired, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, PanelApiConfig, PanelApiQueryParam, ProxyUserCredentials, }, repository::{ - csv_patch_batch_append, csv_patch_batch_remove_expired, csv_patch_batch_sort_by_exp_date, - csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, get_csv_file_path, + compare_alias_exp_date_with_order, csv_patch_batch_append, csv_patch_batch_remove_expired, + csv_patch_batch_sort_by_exp_date, csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, + get_csv_file_path, AliasExpDateSortOrder, }, - utils::{debug_if_enabled, format_http_status, persist_source_config, read_sources_file_from_path}, + utils::{debug_if_enabled, format_http_status, persist_source_config, read_sources_file_from_path, request}, }; use smallvec::SmallVec; -use axum::http::{Method, StatusCode}; -use chrono::{NaiveDateTime, TimeZone}; +use axum::http::{header, HeaderMap, Method, StatusCode}; +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use chrono_tz::Tz; use jsonwebtoken::get_current_timestamp; use log::{error, warn}; @@ -26,10 +27,10 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use shared::{ concat_string, create_bitset, - error::{TuliproxError}, + error::{string_to_io_error, TuliproxError}, model::{ - ConfigInputAliasDto, InputType, PanelApiAliasPoolSizeValue, PanelApiProvisioningMethod, ProxyUserStatus, - SourcesConfigDto, VirtualId, + ConfigInputAliasDto, ConfigInputDto, InputType, PanelApiAliasPoolSizeValue, PanelApiProvisioningMethod, + ProxyUserStatus, SourcesConfigDto, VirtualId, }, utils::{ get_base_url_from_str, get_credentials_from_url, get_credentials_from_url_str, get_i64_from_serde_value, @@ -40,6 +41,7 @@ use shared::{ use std::{ cmp::Ordering, collections::{HashMap, HashSet}, + io, net::SocketAddr, path::{Path, PathBuf}, sync::Arc, @@ -57,6 +59,11 @@ struct AccountCredentials { exp_date: Option, } +const PANEL_API_REQUEST_TIMEOUT_SECS: u64 = 30; +const PANEL_API_RETRY_ATTEMPTS: usize = 2; +const PANEL_API_DEFAULT_RETRY_AFTER_SECS: u64 = 1; +const PANEL_API_MAX_RETRY_AFTER_SECS: u64 = 5; + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] enum PanelApiExpireMode { @@ -497,41 +504,152 @@ fn sanitize_panel_api_json_for_log(value: &Value, sanitize_sensitive: bool) -> V } } +fn panel_api_retryable_status(status: StatusCode) -> bool { + status.is_server_error() + || matches!( + status, + StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY + ) +} + +fn panel_api_retry_after_from_header_value(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(secs) = trimmed.parse::() { + return Some(Duration::from_secs( + secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS), + )); + } + let retry_at = DateTime::parse_from_rfc2822(trimmed).ok()?.with_timezone(&Utc); + let now = Utc::now(); + let secs = retry_at.signed_duration_since(now).num_seconds(); + let secs = if secs <= 0 { + PANEL_API_DEFAULT_RETRY_AFTER_SECS + } else { + u64::try_from(secs).unwrap_or(PANEL_API_DEFAULT_RETRY_AFTER_SECS) + }; + Some(Duration::from_secs( + secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS), + )) +} + +fn panel_api_retry_after_from_headers(headers: &HeaderMap) -> Duration { + headers + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(panel_api_retry_after_from_header_value) + .unwrap_or_else(|| Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) +} + async fn panel_get_json(app_state: &AppState, url: Url) -> Result { let client = app_state.http_client.load(); let sanitized = sanitize_sensitive_info(url.as_str()); - debug_if_enabled!("panel_api request {}", sanitized); - let resp = client - .get(url) - .timeout(std::time::Duration::from_secs(30)) - .send() - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api request failed: {e}")))?; - let status = resp.status(); - let body = resp.text().await.map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api read response failed: {e}")))?; - let json: Value = - serde_json::from_str(&body).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api invalid json (http {status}): {e}")))?; - let sanitize_sensitive = app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); - let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); - if let Ok(json_str) = serde_json::to_string(&json_for_log) { + for attempt in 0..PANEL_API_RETRY_ATTEMPTS { debug_if_enabled!( - "panel_api response (http {}): {}", - format_http_status(status), - sanitize_sensitive_info(&json_str) + "panel_api request attempt {} of {}: {}", + attempt + 1, + PANEL_API_RETRY_ATTEMPTS, + sanitized ); + let resp = client + .get(url.clone()) + .timeout(Duration::from_secs(PANEL_API_REQUEST_TIMEOUT_SECS)) + .send() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api request failed: {e}")))?; + let status = resp.status(); + if !status.is_success() { + if attempt + 1 < PANEL_API_RETRY_ATTEMPTS && panel_api_retryable_status(status) { + let cooldown = panel_api_retry_after_from_headers(resp.headers()); + warn!( + "panel_api request returned {}, retrying after {:.3}s: {}", + format_http_status(status), + cooldown.as_secs_f64(), + sanitized + ); + tokio::time::sleep(cooldown).await; + continue; + } + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api request failed (http {}): {}", + format_http_status(status), + sanitized + ))); + } + let body = resp + .text() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api read response failed: {e}")))?; + let json: Value = serde_json::from_str(&body) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api invalid json (http {status}): {e}")))?; + let sanitize_sensitive = app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); + let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); + if let Ok(json_str) = serde_json::to_string(&json_for_log) { + debug_if_enabled!( + "panel_api response (http {}): {}", + format_http_status(status), + sanitize_sensitive_info(&json_str) + ); + } + return Ok(json); } - Ok(json) + Err(TuliproxError::ConfigPanelApi(format!("panel_api request failed: {sanitized}"))) } -async fn user_api_get_json(app_state: &AppState, url: Url) -> Result { +fn build_user_api_account_info_input_source( + input: &ConfigInput, + username: &str, + password: &str, +) -> Result { + let url = build_player_api_action_url(input.url.as_str(), username, password, "account_info").ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: invalid user_api base_url: {}", + sanitize_sensitive_info(input.url.as_str()) + )) + })?; + + Ok(InputSource::from(input).with_url(url.to_string())) +} + +async fn user_api_get_json(app_state: &AppState, input_source: &InputSource) -> Result { let client = app_state.http_client.load(); - let sanitized = sanitize_sensitive_info(url.as_str()); - debug_if_enabled!("panel_api user_api request {}", sanitized); - let resp = client - .get(url) - .timeout(std::time::Duration::from_secs(30)) - .send() - .await + let url = Url::parse(input_source.url.as_str()).map_err(|e| { + TuliproxError::ConfigPanelApi(format!( + "panel_api user_api invalid url {}: {e}", + sanitize_sensitive_info(input_source.url.as_str()) + )) + })?; + debug_if_enabled!( + "panel_api user_api request {}", + sanitize_sensitive_info(&request::preview_request_diagnostics_for_logging(&url, input_source.get_provider())) + ); + + let config = app_state.app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let headers = request::get_request_headers( + Some(&input_source.headers), + None::<&HashMap>>, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + let resp = request::send_with_retry_and_provider( + &app_state.app_config, + &url, + input_source.get_provider(), + false, + |resolved_url| { + client + .get(resolved_url.clone()) + .headers(headers.clone()) + .timeout(std::time::Duration::from_secs(30)) + }, + ) + .await .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api request failed: {e}")))?; let status = resp.status(); let body = resp.text().await.map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api read response failed: {e}")))?; @@ -642,21 +760,9 @@ async fn fetch_root_user_api_info( return Ok(None); }; - let resolved_url = input.resolve()?; - let base_url = get_base_url_from_str(&resolved_url).unwrap_or_else(|| resolved_url.to_string()); + let input_source = build_user_api_account_info_input_source(input, username.as_ref(), password.as_ref())?; - let Ok(mut url) = Url::parse(base_url.as_str()) else { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: invalid base_url: {}", sanitize_sensitive_info(&base_url)))); - }; - url.set_path("/player_api.php"); - { - let mut pairs = url.query_pairs_mut(); - pairs.append_pair("username", username.as_ref()); - pairs.append_pair("password", password.as_ref()); - pairs.append_pair("action", "account_info"); - } - - let json = match user_api_get_json(app_state, url).await { + let json = match user_api_get_json(app_state, &input_source).await { Ok(json) => json, Err(err) => { debug_if_enabled!( @@ -969,12 +1075,6 @@ fn collect_accounts(input: &ConfigInput) -> Vec { out } -fn compare_alias_exp_date(a: &ConfigInputAliasDto, b: &ConfigInputAliasDto) -> Ordering { - let a_ts = a.exp_date.unwrap_or(i64::MIN); - let b_ts = b.exp_date.unwrap_or(i64::MIN); - b_ts.cmp(&a_ts).then_with(|| a.name.cmp(&b.name)) -} - fn compare_alias_exp_date_config(a: &ConfigInputAlias, b: &ConfigInputAlias) -> Ordering { let a_ts = a.exp_date.unwrap_or(i64::MIN); let b_ts = b.exp_date.unwrap_or(i64::MIN); @@ -994,12 +1094,12 @@ fn aliases_need_sort_config(aliases: &[ConfigInputAlias]) -> bool { aliases.windows(2).any(|pair| compare_alias_exp_date_config(&pair[0], &pair[1]) == Ordering::Greater) } -fn sort_aliases_by_exp_date(aliases: &mut Vec) -> bool { +fn sort_aliases_by_exp_date_order(aliases: &mut Vec, order: AliasExpDateSortOrder) -> bool { if aliases.len() < 2 { return false; } let mut sorted = aliases.clone(); - sorted.sort_by(compare_alias_exp_date); + sorted.sort_by(|a, b| compare_alias_exp_date_with_order(a, b, order)); if &sorted == aliases { false } else { @@ -1052,6 +1152,40 @@ fn should_reload_sources_after_internal_write(app_state: &AppState) -> bool { !app_state.app_config.config.load().config_hot_reload } +fn append_sources_yml_alias( + input_name: &Arc, + input: &mut ConfigInputDto, + alias_name: Arc, + base_url: String, + username: String, + password: String, + exp_date: Option, +) -> Result { + let input_type = input.input_type; + let aliases = input.aliases.get_or_insert_with(Vec::new); + let next_index = aliases.iter().map(|alias| alias.id).max().unwrap_or(0); + if next_index == u16::MAX { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: cannot add alias for '{input_name}': alias id overflow" + ))); + } + + let mut alias = ConfigInputAliasDto { + id: 0, + name: alias_name, + url: base_url, + username: Some(username), + password: Some(password), + priority: 0, + max_connections: 1, + exp_date, + enabled: true, + }; + alias.prepare(next_index, &input_type)?; + aliases.push(alias); + Ok(aliases.len().saturating_sub(1)) +} + pub(crate) fn is_alias_pool_max_reached(app_state: &AppState, input: &ConfigInput) -> bool { let Some(panel_cfg) = input.panel_api.as_ref() else { return false; @@ -1116,50 +1250,6 @@ pub(crate) fn find_input_by_provider_name(app_state: &AppState, provider_name: & None } -#[allow(clippy::too_many_arguments)] -async fn patch_source_yml_add_alias( - app_state: &Arc, - source_file_path: &Path, - input_name: &Arc, - alias_name: &Arc, - base_url: &str, - username: &str, - password: &str, - exp_date: Option, -) -> Result<(), TuliproxError> { - let mut sources = match read_sources_file_from_path(source_file_path, false, false, None).await { - Ok(sources) => sources, - Err(e) => return Err(TuliproxError::ConfigPanelApi(format!("panel_api: failed to read source file: {e}"))), - }; - - let Some(input) = sources.inputs.iter_mut().find(|i| i.name == *input_name) else { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml"))); - }; - - let aliases = input.aliases.get_or_insert_with(Vec::new); - let next_index = aliases.iter().map(|a| a.id).max().unwrap_or(0); - if next_index == u16::MAX { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: cannot add alias for '{input_name}': alias id overflow"))); - } - - let alias = ConfigInputAliasDto { - id: 0, - name: Arc::clone(alias_name), - url: base_url.to_string(), - username: Some(username.to_string()), - password: Some(password.to_string()), - priority: 0, - max_connections: 1, - exp_date, - enabled: true, - }; - - input.upsert_alias(alias)?; - - persist_source_config(app_state, Some(source_file_path), sources).await?; - Ok(()) -} - #[derive(Debug, Clone)] enum SourcesYmlPatch { UpdatePanelApiCredits { @@ -1168,6 +1258,7 @@ enum SourcesYmlPatch { }, SortAliases { input_name: Arc, + order: AliasExpDateSortOrder, }, UpdateExpDate { input_name: Arc, @@ -1180,6 +1271,12 @@ enum SourcesYmlPatch { password: String, exp_date: Option, }, + PersistProvisionedAccount { + input_name: Arc, + username: String, + password: String, + exp_date: Option, + }, UpdateAliasCredentials { input_name: Arc, alias_name: Arc, @@ -1303,14 +1400,14 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa changed = true; } } - SourcesYmlPatch::SortAliases { input_name } => { + SourcesYmlPatch::SortAliases { input_name, order } => { let idx = *inputs_by_name .get(input_name.as_ref()) .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { continue; }; - if sort_aliases_by_exp_date(aliases) { + if sort_aliases_by_exp_date_order(aliases, *order) { alias_indices[idx] = aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); changed = true; @@ -1321,7 +1418,10 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa .get(input_name.as_ref()) .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; if account_name == input_name { - if doc.inputs[idx].exp_date != Some(*exp_date) { + if doc.inputs[idx].exp_date != Some(*exp_date) + || !doc.inputs[idx].enabled + || doc.inputs[idx].max_connections != 1 + { doc.inputs[idx].exp_date = Some(*exp_date); doc.inputs[idx].enabled = true; doc.inputs[idx].max_connections = 1; @@ -1338,13 +1438,18 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa .aliases .as_mut() .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")))?; - if aliases[alias_idx].exp_date != Some(*exp_date) { + if aliases[alias_idx].exp_date != Some(*exp_date) || aliases[alias_idx].max_connections != 1 { aliases[alias_idx].exp_date = Some(*exp_date); aliases[alias_idx].max_connections = 1; changed = true; } } - SourcesYmlPatch::UpdateRootCredentials { input_name, username, password, exp_date } => { + SourcesYmlPatch::UpdateRootCredentials { + input_name, + username, + password, + exp_date, + } => { let idx = *inputs_by_name .get(input_name.as_ref()) .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; @@ -1365,6 +1470,64 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa changed = true; } } + SourcesYmlPatch::PersistProvisionedAccount { input_name, username, password, exp_date } => { + let idx = *inputs_by_name + .get(input_name.as_ref()) + .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; + let input = &mut doc.inputs[idx]; + let current_root_is_usable = + input.exp_date.is_some() && !is_input_expired_at(input.exp_date, get_current_timestamp()); + if current_root_is_usable { + let mut existing_names = vec![input.name.clone()]; + if let Some(aliases) = input.aliases.as_ref() { + existing_names.extend(aliases.iter().map(|alias| alias.name.clone())); + } + let alias_name = derive_unique_alias_name(&existing_names, &input.name, username); + let base_url = resolve_provisioned_account_base_url(input.url.as_str(), None, username, password); + let alias_idx = append_sources_yml_alias( + input_name, + input, + Arc::clone(&alias_name), + base_url, + username.clone(), + password.clone(), + *exp_date, + )?; + alias_indices[idx].insert(Arc::clone(&alias_name), alias_idx); + if let Some(aliases) = input.aliases.as_mut() { + if sort_aliases_by_exp_date_order(aliases, AliasExpDateSortOrder::NewestFirst) { + alias_indices[idx] = + aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); + } + } + debug_if_enabled!( + "panel_api preserved current root for input {} and added provisioned account as alias {}", + sanitize_sensitive_info(input_name.as_ref()), + sanitize_sensitive_info(alias_name.as_ref()) + ); + changed = true; + continue; + } + debug_if_enabled!( + "panel_api stored provisioned account as root for input {} because current root is expired or missing exp_date", + sanitize_sensitive_info(input_name.as_ref()) + ); + let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; + if input.username.as_deref() != Some(username.as_str()) + || input.password.as_deref() != Some(password.as_str()) + || exp_date_changed + { + input.username = Some(username.clone()); + input.password = Some(password.clone()); + input.enabled = true; + input.max_connections = 1; + if let Some(exp_date) = *exp_date { + input.exp_date = Some(exp_date); + } + update_url_query_credentials_if_present(&mut input.url, username, password); + changed = true; + } + } SourcesYmlPatch::UpdateAliasCredentials { input_name, alias_name, username, password, exp_date } => { let idx = *inputs_by_name .get(input_name.as_ref()) @@ -1398,27 +1561,16 @@ fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPa let idx = *inputs_by_name .get(input_name) .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml")))?; - let input_type = doc.inputs[idx].input_type; - let aliases = doc.inputs[idx].aliases.get_or_insert_with(Vec::new); - - let next_index = u16::try_from(aliases.len()) - .map_err(|_| TuliproxError::ConfigPanelApi(format!("panel_api: cannot add alias for '{input_name}': alias id overflow")))?; - - let mut alias = ConfigInputAliasDto { - id: 0, - name: Arc::clone(alias_name), - url: base_url.clone(), - username: Some(username.clone()), - password: Some(password.clone()), - priority: 0, - max_connections: 1, - exp_date: *exp_date, - enabled: true, - }; - alias.prepare(next_index, &input_type)?; - aliases.push(alias); - - alias_indices[idx].insert(Arc::clone(alias_name), aliases.len().saturating_sub(1)); + let alias_idx = append_sources_yml_alias( + input_name, + &mut doc.inputs[idx], + Arc::clone(alias_name), + base_url.clone(), + username.clone(), + password.clone(), + *exp_date, + )?; + alias_indices[idx].insert(Arc::clone(alias_name), alias_idx); changed = true; } SourcesYmlPatch::RemoveExpiredAliases { input_name } => { @@ -1463,44 +1615,6 @@ async fn persist_sources_yml_with_patches( Ok(true) } -async fn patch_source_yml_update_exp_date( - app_state: &Arc, - source_file_path: &Path, - input_name: &Arc, - account_name: &Arc, - exp_date: i64, -) -> Result<(), TuliproxError> { - let mut sources = match read_sources_file_from_path(source_file_path, false, false, None).await { - Ok(sources) => sources, - Err(e) => return Err(TuliproxError::ConfigPanelApi(format!("panel_api: failed to read source file: {e}"))), - }; - - let Some(input) = sources.inputs.iter_mut().find(|i| i.name == *input_name) else { - return Err(TuliproxError::ConfigPanelApi(format!("panel_api: could not find input '{input_name}' in source.yml"))); - }; - - if account_name == input_name { - input.exp_date = Some(exp_date); - input.enabled = true; - input.max_connections = 1; - } else if let Some(aliases) = input.aliases.as_mut() { - let Some(alias) = aliases.iter_mut().find(|a| &a.name == account_name) else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find alias '{account_name}' under input '{input_name}' in source.yml" - ))); - }; - alias.exp_date = Some(exp_date); - alias.max_connections = 1; - } else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: input '{input_name}' has no aliases; cannot update exp_date for '{account_name}'" - ))); - } - - persist_source_config(app_state, Some(source_file_path), sources).await?; - Ok(()) -} - const MAX_ALIAS_NAME_ATTEMPTS: usize = 1000; fn derive_unique_alias_name(existing: &[Arc], input_name: &Arc, username: &str) -> Arc { @@ -1537,28 +1651,21 @@ fn derive_unique_alias_name_set(existing: &HashSet>, input_name: &Arc (&str, &str) { - match self { - Self::Renewed { username, password } | Self::Created { username, password } => { - (username.as_str(), password.as_str()) - } - } - } - pub(crate) fn kind_label(&self) -> &'static str { match self { - Self::Renewed { .. } => "client_renew", - Self::Created { .. } => "client_new", + Self::Renewed => "client_renew", + Self::Created => "client_new", } } } #[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_arguments)] async fn try_renew_expired_account( app_state: &Arc, input: &ConfigInput, @@ -1566,6 +1673,7 @@ async fn try_renew_expired_account( is_batch: bool, sources_path: &Path, treat_missing_exp_date_as_expired: bool, + include_root: bool, optional: PanelApiOptionalFlagsSet, ) -> Option { if !optional.contains(PanelApiOptionalFlags::ClientRenew) { @@ -1573,6 +1681,9 @@ async fn try_renew_expired_account( } let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); let mut candidates = collect_accounts(input); + if !include_root { + candidates.retain(|acct| acct.name != input.name); + } for acct in &mut candidates { if treat_missing_exp_date_as_expired && acct.exp_date.is_none() { acct.exp_date = @@ -1631,13 +1742,27 @@ async fn try_renew_expired_account( { debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); } + if let Err(err) = + csv_patch_batch_sort_by_exp_date(input.input_type, &csv_path, AliasExpDateSortOrder::NewestFirst) + .await + { + debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); + } } } else { let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; - if let Err(err) = - patch_source_yml_update_exp_date(app_state, sources_path, &input.name, &acct.name, new_exp) - .await - { + let patches = [ + SourcesYmlPatch::UpdateExpDate { + input_name: input.name.clone(), + account_name: Arc::clone(&acct.name), + exp_date: new_exp, + }, + SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &patches).await { debug_if_enabled!("panel_api failed to persist renew exp_date to source.yml: {}", err); } } @@ -1648,10 +1773,7 @@ async fn try_renew_expired_account( debug_if_enabled!("panel_api reload sources failed: {}", err); } } - return Some(PanelApiProvisionOutcome::Renewed { - username: acct.username.clone(), - password: acct.password.clone(), - }); + return Some(PanelApiProvisionOutcome::Renewed); } Err(err) => { debug_if_enabled!( @@ -1665,6 +1787,190 @@ async fn try_renew_expired_account( None } +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn try_refresh_root_account_on_exhausted( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, + optional: PanelApiOptionalFlagsSet, +) -> Option { + let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); + let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); + if !renew_enabled && !new_enabled { + return None; + } + + let now = get_current_timestamp(); + let offset_secs = panel_cfg + .provisioning + .offset + .as_deref() + .and_then(|value| parse_panel_api_provisioning_offset_secs(value).ok()) + .unwrap_or(0); + let root_exp_missing = input.exp_date.is_none(); + let root_expired = is_input_expired_at(input.exp_date, now); + let root_expiring = is_expiring_with_offset_at(input.exp_date, offset_secs, now); + if !root_exp_missing && !root_expired && !root_expiring { + return None; + } + + let old_username = input.username.clone().unwrap_or_default(); + let old_password = input.password.clone().unwrap_or_default(); + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + + let (outcome, active_username, active_password, credentials_changed) = + if renew_enabled && !old_username.is_empty() && !old_password.is_empty() { + match panel_client_renew(app_state, panel_cfg, old_username.as_str(), old_password.as_str()).await { + Ok(()) => { + (PanelApiProvisionOutcome::Renewed, old_username.clone(), old_password.clone(), false) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + if !new_enabled { + return None; + } + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, _base_url_from_resp)) => { + (PanelApiProvisionOutcome::Created, username, password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + } + } + } + } else if new_enabled { + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, _base_url_from_resp)) => { + (PanelApiProvisionOutcome::Created, username, password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + } + } else { + return None; + }; + + if adult_enabled { + if let Err(err) = + panel_client_adult_content(app_state, panel_cfg, Some((active_username.as_str(), active_password.as_str()))) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + if !wait_for_panel_api_account_ready( + app_state, + input, + panel_cfg, + input.name.as_ref(), + active_username.as_str(), + active_password.as_str(), + ) + .await + { + debug_if_enabled!( + "panel_api root account not ready after probe/cooldown for {}", + sanitize_sensitive_info(&input.name) + ); + return None; + } + + let refreshed_exp_date = panel_client_info(app_state, panel_cfg, active_username.as_str(), active_password.as_str(), None) + .await + .ok() + .flatten(); + let exp_date = if credentials_changed { + refreshed_exp_date + } else { + refreshed_exp_date.or(input.exp_date) + }; + + if is_batch { + let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); + let Ok(csv_path) = get_csv_file_path(batch_url) else { + return None; + }; + let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + let result = if credentials_changed { + csv_patch_batch_update_credentials( + input.input_type, + &csv_path, + &input.name, + old_username.as_str(), + old_password.as_str(), + active_username.as_str(), + active_password.as_str(), + exp_date, + ) + .await + } else if let Some(exp_date) = exp_date { + csv_patch_batch_update_exp_date( + input.input_type, + &csv_path, + &input.name, + active_username.as_str(), + active_password.as_str(), + exp_date, + ) + .await + } else { + Ok(()) + }; + if let Err(err) = result { + debug_if_enabled!("panel_api failed to persist root provisioning to csv: {}", err); + return None; + } + if let Err(err) = csv_patch_batch_sort_by_exp_date(input.input_type, &csv_path, AliasExpDateSortOrder::NewestFirst).await { + debug_if_enabled!("panel_api failed to sort csv accounts after root provisioning: {}", err); + return None; + } + } else { + let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; + let patch = SourcesYmlPatch::PersistProvisionedAccount { + input_name: input.name.clone(), + username: active_username, + password: active_password, + exp_date, + }; + if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &[patch]).await { + debug_if_enabled!("panel_api failed to persist root provisioning to source.yml: {}", err); + return None; + } + } + + if should_reload_sources_after_internal_write(app_state.as_ref()) { + if let Err(err) = ConfigFile::load_sources(app_state).await { + debug_if_enabled!("panel_api reload sources failed: {}", err); + return None; + } + } + + Some(outcome) +} + #[allow(clippy::too_many_lines)] async fn try_create_new_account( app_state: &Arc, @@ -1705,6 +2011,23 @@ async fn try_create_new_account( let exp_date = panel_client_info(app_state, panel_cfg, &username, &password, None).await.ok().flatten(); + if !wait_for_panel_api_account_ready( + app_state, + input, + panel_cfg, + alias_name.as_ref(), + username.as_str(), + password.as_str(), + ) + .await + { + debug_if_enabled!( + "panel_api client_new account not ready after probe/cooldown for {}", + sanitize_sensitive_info(alias_name.as_ref()) + ); + return None; + } + if is_batch { let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); match get_csv_file_path(batch_url) { @@ -1729,6 +2052,13 @@ async fn try_create_new_account( warn!("panel_api failed to append new account to csv: {err}"); return None; } + if let Err(err) = + csv_patch_batch_sort_by_exp_date(batch_type, &csv_path, AliasExpDateSortOrder::NewestFirst) + .await + { + warn!("panel_api failed to sort csv accounts after append: {err}"); + return None; + } } Err(err) => { warn!( @@ -1741,18 +2071,21 @@ async fn try_create_new_account( } } else { let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; - if let Err(err) = patch_source_yml_add_alias( - app_state, - sources_path, - &input.name, - &alias_name, - &base_url, - &username, - &password, - exp_date, - ) - .await - { + let patches = [ + SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: Arc::clone(&alias_name), + base_url, + username, + password, + exp_date, + }, + SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &patches).await { warn!("panel_api failed to persist new alias to source.yml: {err}"); return None; } @@ -1764,7 +2097,7 @@ async fn try_create_new_account( return None; } } - Some(PanelApiProvisionOutcome::Created { username, password }) + Some(PanelApiProvisionOutcome::Created) } Err(err) => { debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(&err.to_string())); @@ -1775,8 +2108,22 @@ async fn try_create_new_account( pub async fn try_provision_account_on_exhausted( app_state: &Arc, - input: &ConfigInput, + input_name: &Arc, ) -> Option { + let _input_lock = app_state + .app_config + .file_locks + .write_lock_str(format!("panel_api:{input_name}").as_str()) + .await; + let current_input = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()); + let Some(input) = current_input.as_deref() else { + debug_if_enabled!( + "panel_api: skipped (input no longer exists) for input {}", + sanitize_sensitive_info(input_name.as_ref()) + ); + return None; + }; + let Some(panel_cfg) = input.panel_api.as_ref() else { debug_if_enabled!( "panel_api: skipped (no panel_api config) for input {}", @@ -1799,17 +2146,10 @@ pub async fn try_provision_account_on_exhausted( return None; } - let _input_lock = - app_state.app_config.file_locks.write_lock_str(format!("panel_api:{}", input.name).as_str()).await; - if let Err(err) = validate_panel_api_config(panel_cfg) { debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); return None; } - if is_alias_pool_max_reached(app_state, input) { - return None; - } - let optional = resolve_panel_api_optional_flags(panel_cfg, &input.name); debug_if_enabled!( "panel_api: exhausted -> provisioning for input {} (aliases={})", @@ -1822,7 +2162,23 @@ pub async fn try_provision_account_on_exhausted( let sources_path = PathBuf::from(&sources_file_path); if let Some(outcome) = - try_renew_expired_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), true, optional).await + try_refresh_root_account_on_exhausted(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional).await + { + debug_if_enabled!( + "panel_api: provisioning succeeded via root {} for input {}", + outcome.kind_label(), + sanitize_sensitive_info(&input.name) + ); + return Some(outcome); + } + + if is_alias_pool_max_reached(app_state, input) { + return None; + } + + if let Some(outcome) = + try_renew_expired_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), true, false, optional) + .await { debug_if_enabled!( "panel_api: provisioning succeeded via client_renew for input {}", @@ -2085,6 +2441,7 @@ async fn sync_panel_api_for_input_on_boot( let csv_path = if is_batch { get_csv_file_path(batch_url).ok() } else { None }; let mut sources_yml_patches: Vec = Vec::new(); let mut pending_sources_yml = false; + let mut source_yml_sort_aliases_requested = false; let mut accounts = collect_accounts(input.as_ref()); if panel_cfg.alias_pool.is_some() { @@ -2103,8 +2460,12 @@ async fn sync_panel_api_for_input_on_boot( && csv_path.is_none() && input.aliases.as_ref().is_some_and(|aliases| aliases_need_sort_config(aliases)) { - sources_yml_patches.push(SourcesYmlPatch::SortAliases { input_name: input.name.clone() }); + sources_yml_patches.push(SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }); pending_sources_yml = true; + source_yml_sort_aliases_requested = true; } if let Some((root_username, root_password)) = extract_account_creds_from_input(input.as_ref()) { @@ -3202,7 +3563,7 @@ async fn sync_panel_api_for_input_on_boot( if panel_cfg.alias_pool.is_some() { if let Some(csv_path) = csv_path.as_ref() { let _csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - match csv_patch_batch_sort_by_exp_date(input.input_type, csv_path).await { + match csv_patch_batch_sort_by_exp_date(input.input_type, csv_path, AliasExpDateSortOrder::NewestFirst).await { Ok(true) => any_change = true, Ok(false) => {} Err(err) => debug_if_enabled!( @@ -3211,6 +3572,11 @@ async fn sync_panel_api_for_input_on_boot( err ), } + } else if pending_sources_yml && !source_yml_sort_aliases_requested { + sources_yml_patches.push(SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }); } } @@ -3325,12 +3691,8 @@ fn build_player_api_action_url(base_url: &str, username: &str, password: &str, a Some(test_url) } -fn build_panel_api_test_url(base_url: &str, username: &str, password: &str) -> Option { - build_player_api_action_url(base_url, username, password, "account_info") -} - enum PanelApiProbeTarget { - PlayerApi { action: &'static str, url: Url }, + PlayerApi { action: &'static str, input_source: InputSource }, } impl PanelApiProbeTarget { @@ -3356,7 +3718,10 @@ fn build_panel_api_probe_targets(input: &ConfigInput, username: &str, password: let mut targets = Vec::new(); for action in ["client_info", "get_live_categories", "get_series_categories", "get_vod_categories"] { if let Some(url) = build_player_api_action_url(input.url.as_str(), username, password, action) { - targets.push(PanelApiProbeTarget::PlayerApi { action, url }); + targets.push(PanelApiProbeTarget::PlayerApi { + action, + input_source: InputSource::from(input).with_url(url.to_string()), + }); } } targets @@ -3374,31 +3739,31 @@ async fn probe_panel_api_targets( continue; } match target { - PanelApiProbeTarget::PlayerApi { action, url } => { - match probe_panel_api_test_url(app_state, url, probe_method).await { + PanelApiProbeTarget::PlayerApi { action, input_source } => { + match probe_panel_api_test_url(app_state, input_source, probe_method).await { Ok(status) => { debug_if_enabled!( "panel_api probe status: '{}' action={} url: {}", format_http_status(status), action, - sanitize_sensitive_info(url.as_str()) + sanitize_sensitive_info(input_source.url.as_str()) ); if status.is_success() { done.insert(action); } } Err(err) => { - if err.is_timeout() { + if err.kind() == io::ErrorKind::TimedOut { debug_if_enabled!( "panel_api probe timeout action={} url: {}", action, - sanitize_sensitive_info(url.as_str()) + sanitize_sensitive_info(input_source.url.as_str()) ); } else { debug_if_enabled!( "panel_api probe failed action={} url: {}: {err}", action, - sanitize_sensitive_info(url.as_str()) + sanitize_sensitive_info(input_source.url.as_str()) ); } } @@ -3411,12 +3776,38 @@ async fn probe_panel_api_targets( async fn probe_panel_api_test_url( app_state: &Arc, - test_url: &Url, + input_source: &InputSource, method: PanelApiProvisioningMethod, -) -> Result { +) -> Result { let client = app_state.http_client.load(); let request_method = provisioning_method_to_reqwest(method); - let response = client.request(request_method, test_url.clone()).send().await?; + let test_url = Url::parse(input_source.url.as_str()).map_err(|err| { + string_to_io_error(format!( + "Malformed URL {}: {}", + sanitize_sensitive_info(input_source.url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + )) + })?; + + let config = app_state.app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let headers = request::get_request_headers( + Some(&input_source.headers), + None::<&HashMap>>, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + let response = request::send_with_retry_and_provider( + &app_state.app_config, + &test_url, + input_source.get_provider(), + false, + |resolved_url| client.request(request_method.clone(), resolved_url.clone()).headers(headers.clone()), + ) + .await?; Ok(response.status()) } @@ -3434,7 +3825,7 @@ async fn apply_provisioning_cooldown(panel_cfg: &PanelApiConfig, account_name: & tokio::time::sleep(Duration::from_secs(cooldown_secs)).await; } -async fn wait_for_panel_api_account_ready( +pub(crate) async fn wait_for_panel_api_account_ready( app_state: &Arc, input: &ConfigInput, panel_cfg: &PanelApiConfig, @@ -3495,7 +3886,7 @@ async fn wait_for_panel_api_account_ready( #[allow(clippy::too_many_lines)] pub(crate) async fn run_panel_api_provisioning_probe( app_state: Arc, - input: ConfigInput, + input_name: Arc, stop_signal: CancellationToken, addr: SocketAddr, virtual_id: VirtualId, @@ -3504,6 +3895,23 @@ pub(crate) async fn run_panel_api_provisioning_probe( return Ok(()); } let provisioning_kick_secs = 0; + let Some(input) = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()) else { + debug_if_enabled!( + "panel_api provisioning probe skipped (input no longer exists) for input {}", + sanitize_sensitive_info(input_name.as_ref()) + ); + stop_signal.cancel(); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + }; let Some(panel_cfg) = input.panel_api.as_ref() else { debug_if_enabled!( @@ -3558,23 +3966,17 @@ pub(crate) async fn run_panel_api_provisioning_probe( } let max_wait_secs = panel_cfg.provisioning.timeout_sec; - let probe_interval_secs = panel_cfg.provisioning.probe_interval_sec.max(1); - let probe_method = panel_cfg.provisioning.method; debug_if_enabled!( - "panel_api provisioning probe start for input {} (timeout={}s interval={}s method={})", + "panel_api provisioning probe start for input {} (timeout={}s)", sanitize_sensitive_info(&input.name), - max_wait_secs, - probe_interval_secs, - probe_method + max_wait_secs ); - let deadline = Instant::now() + Duration::from_secs(max_wait_secs); let outcome = tokio::select! { () = stop_signal.cancelled() => return Ok(()), - outcome = try_provision_account_on_exhausted(&app_state, &input) => outcome, + outcome = try_provision_account_on_exhausted(&app_state, &input_name) => outcome, }; - let credentials = outcome.as_ref().map(PanelApiProvisionOutcome::credentials); if let Some(outcome) = outcome.as_ref() { debug_if_enabled!( @@ -3589,7 +3991,7 @@ pub(crate) async fn run_panel_api_provisioning_probe( ); } - let Some((username, password)) = credentials else { + if outcome.is_none() { if max_wait_secs > 0 { tokio::select! { () = stop_signal.cancelled() => return Ok(()), @@ -3612,100 +4014,12 @@ pub(crate) async fn run_panel_api_provisioning_probe( &addr, virtual_id, provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; + DisconnectReason::Provisioning, + ) + .await; return Ok(()); - }; - - let resolved_url = input.resolve()?; - - let Some(test_url) = build_panel_api_test_url(&resolved_url, username, password) else { - if max_wait_secs > 0 { - tokio::select! { - () = stop_signal.cancelled() => return Ok(()), - () = tokio::time::sleep(Duration::from_secs(max_wait_secs)) => {} - } - } - debug_if_enabled!( - "panel_api provisioning probe failed to build test url for input {}", - sanitize_sensitive_info(&input.name) - ); - stop_signal.cancel(); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - }; - - let probe_delay = Duration::from_secs(probe_interval_secs); - let mut attempt = 0u64; - let mut ready = false; - while Instant::now() < deadline { - attempt += 1; - debug_if_enabled!("panel_api provisioning probe attempt {}", attempt); - let probe_result = tokio::select! { - () = stop_signal.cancelled() => return Ok(()), - probe_result = probe_panel_api_test_url(&app_state, &test_url, probe_method) => probe_result, - }; - match probe_result { - Ok(status) => { - debug_if_enabled!( - "panel_api provisioning probe status: '{}' url: {}", - format_http_status(status), - sanitize_sensitive_info(test_url.as_str()) - ); - if status.is_success() { - ready = true; - break; - } - } - Err(err) => { - if err.is_timeout() { - debug_if_enabled!( - "panel_api provisioning probe timeout for {}", - sanitize_sensitive_info(test_url.as_str()) - ); - } else { - debug_if_enabled!( - "panel_api provisioning probe failed for {}: {err}", - sanitize_sensitive_info(test_url.as_str()) - ); - } - } - } - - let now = Instant::now(); - if now >= deadline { - break; - } - let remaining = deadline.checked_duration_since(now).unwrap_or_default(); - let sleep_for = if remaining < probe_delay { remaining } else { probe_delay }; - tokio::select! { - () = stop_signal.cancelled() => return Ok(()), - () = tokio::time::sleep(sleep_for) => {} - } } - if ready { - debug_if_enabled!( - "panel_api provisioning ready for input {} (attempts={})", - sanitize_sensitive_info(&input.name), - attempt - ); - } else { - debug_if_enabled!( - "panel_api provisioning probe timeout reached for input {} (attempts={})", - sanitize_sensitive_info(&input.name), - attempt - ); - } if stop_signal.is_cancelled() { return Ok(()); } @@ -3765,11 +4079,11 @@ pub fn create_panel_api_provisioning_stream_details( } let app_state_clone = Arc::clone(app_state); - let input_clone = input.clone(); + let input_name = Arc::clone(&input.name); let stop_clone = stop_signal.clone(); tokio::spawn(async move { if let Err(err) = - run_panel_api_provisioning_probe(app_state_clone, input_clone, stop_clone, addr, virtual_id).await + run_panel_api_provisioning_probe(app_state_clone, input_name, stop_clone, addr, virtual_id).await { error!("Error running Probe: {err:?}"); } @@ -3792,9 +4106,256 @@ pub fn create_panel_api_provisioning_stream_details( #[cfg(test)] mod tests { - use super::resolve_provisioned_account_base_url; + use super::{ + apply_sources_yml_patches, build_panel_api_probe_targets, build_user_api_account_info_input_source, + panel_api_retry_after_from_header_value, panel_api_retryable_status, resolve_provisioned_account_base_url, + AliasExpDateSortOrder, PanelApiProbeTarget, SourcesYmlPatch, PANEL_API_DEFAULT_RETRY_AFTER_SECS, + PANEL_API_MAX_RETRY_AFTER_SECS, + }; + use axum::http::StatusCode; + use crate::model::{ConfigInput, ConfigProvider}; + use shared::model::{ConfigInputAliasDto, ConfigInputDto, ConfigProviderDto, InputType, ProviderUrlSelectionPolicy, SourcesConfigDto}; + use std::time::Duration; + use std::sync::Arc; use url::Url; + fn source_alias(name: &str, exp_date: Option) -> ConfigInputAliasDto { + ConfigInputAliasDto { + id: 0, + name: Arc::from(name), + url: "provider://tivione".to_string(), + username: Some(format!("{name}-user")), + password: Some(format!("{name}-pass")), + priority: 0, + max_connections: 1, + exp_date, + enabled: true, + } + } + + fn source_doc_with_aliases(aliases: Vec) -> SourcesConfigDto { + SourcesConfigDto { + inputs: vec![ConfigInputDto { + name: Arc::from("cdn-dev"), + input_type: InputType::Xtream, + url: "provider://tivione".to_string(), + username: Some("root-user".to_string()), + password: Some("root-pass".to_string()), + aliases: Some(aliases), + ..ConfigInputDto::default() + }], + ..SourcesConfigDto::default() + } + } + + #[test] + fn sources_yml_add_alias_appends_without_overwriting_existing_alias() { + let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(10))]); + doc.inputs[0].aliases.as_mut().expect("aliases")[0].id = 2; + + let changed = apply_sources_yml_patches( + &mut doc, + &[ + SourcesYmlPatch::AddAlias { + input_name: Arc::from("cdn-dev"), + alias_name: Arc::from("cdn-dev-new"), + base_url: "provider://tivione".to_string(), + username: "new-user".to_string(), + password: "new-pass".to_string(), + exp_date: Some(20), + }, + SourcesYmlPatch::SortAliases { + input_name: Arc::from("cdn-dev"), + order: AliasExpDateSortOrder::NewestFirst, + }, + ], + ) + .expect("patches apply"); + + assert!(changed); + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases.len(), 2); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new"); + assert_eq!(aliases[0].id, 3); + assert_eq!(aliases[1].name.as_ref(), "cdn-dev-old"); + assert_eq!(aliases[1].id, 2); + } + + #[test] + fn sources_yml_sort_aliases_oldest_first_is_available_for_maintenance_paths() { + let mut doc = source_doc_with_aliases(vec![ + source_alias("cdn-dev-newest", Some(30)), + source_alias("cdn-dev-oldest", Some(10)), + source_alias("cdn-dev-missing-exp", None), + ]); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::SortAliases { + input_name: Arc::from("cdn-dev"), + order: AliasExpDateSortOrder::OldestFirst, + }], + ) + .expect("patches apply"); + + assert!(changed); + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-oldest"); + assert_eq!(aliases[1].name.as_ref(), "cdn-dev-newest"); + assert_eq!(aliases[2].name.as_ref(), "cdn-dev-missing-exp"); + } + + #[test] + fn sources_yml_update_exp_date_keeps_legacy_root_refresh_semantics() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].exp_date = Some(20); + doc.inputs[0].enabled = false; + doc.inputs[0].max_connections = 0; + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdateExpDate { + input_name: Arc::from("cdn-dev"), + account_name: Arc::from("cdn-dev"), + exp_date: 20, + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].exp_date, Some(20)); + assert!(doc.inputs[0].enabled); + assert_eq!(doc.inputs[0].max_connections, 1); + } + + #[test] + fn sources_yml_update_exp_date_keeps_legacy_alias_refresh_semantics() { + let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(20))]); + doc.inputs[0].aliases.as_mut().expect("aliases")[0].max_connections = 0; + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdateExpDate { + input_name: Arc::from("cdn-dev"), + account_name: Arc::from("cdn-dev-old"), + exp_date: 20, + }], + ) + .expect("patches apply"); + + assert!(changed); + let alias = &doc.inputs[0].aliases.as_ref().expect("aliases")[0]; + assert_eq!(alias.exp_date, Some(20)); + assert_eq!(alias.max_connections, 1); + } + + #[test] + fn sources_yml_update_root_credentials_updates_root_directly() { + let mut doc = source_doc_with_aliases(Vec::new()); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdateRootCredentials { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); + assert_eq!(doc.inputs[0].exp_date, Some(42)); + assert!(doc.inputs[0].enabled); + assert_eq!(doc.inputs[0].max_connections, 1); + } + + #[test] + fn sources_yml_persist_provisioned_account_adds_alias_when_current_root_is_valid() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].username = Some("current-root".to_string()); + doc.inputs[0].password = Some("current-pass".to_string()); + doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") + 3600); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::PersistProvisionedAccount { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("current-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("current-pass")); + assert_ne!(doc.inputs[0].exp_date, Some(42)); + + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases.len(), 1); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new-root"); + assert_eq!(aliases[0].username.as_deref(), Some("new-root")); + assert_eq!(aliases[0].password.as_deref(), Some("new-pass")); + assert_eq!(aliases[0].exp_date, Some(42)); + } + + #[test] + fn sources_yml_persist_provisioned_account_replaces_root_when_current_root_is_expired() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].username = Some("current-root".to_string()); + doc.inputs[0].password = Some("current-pass".to_string()); + doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") - 1); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::PersistProvisionedAccount { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); + assert_eq!(doc.inputs[0].exp_date, Some(42)); + assert!(doc.inputs[0].aliases.as_ref().is_none_or(Vec::is_empty)); + } + + #[test] + fn panel_api_retryable_status_covers_rate_limit_and_temporary_failures() { + assert!(panel_api_retryable_status(StatusCode::TOO_MANY_REQUESTS)); + assert!(panel_api_retryable_status(StatusCode::REQUEST_TIMEOUT)); + assert!(panel_api_retryable_status(StatusCode::TOO_EARLY)); + assert!(panel_api_retryable_status(StatusCode::BAD_GATEWAY)); + assert!(!panel_api_retryable_status(StatusCode::BAD_REQUEST)); + assert!(!panel_api_retryable_status(StatusCode::UNAUTHORIZED)); + assert!(!panel_api_retryable_status(StatusCode::NOT_FOUND)); + } + + #[test] + fn panel_api_retry_after_header_is_short_and_bounded() { + assert_eq!( + panel_api_retry_after_from_header_value("2"), + Some(Duration::from_secs(2)) + ); + assert_eq!( + panel_api_retry_after_from_header_value("0"), + Some(Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) + ); + assert_eq!( + panel_api_retry_after_from_header_value("600"), + Some(Duration::from_secs(PANEL_API_MAX_RETRY_AFTER_SECS)) + ); + assert_eq!(panel_api_retry_after_from_header_value("not-a-delay"), None); + } + #[test] fn resolve_base_url_keeps_provider_scheme_for_provisioned_accounts() { let result = resolve_provisioned_account_base_url( @@ -3859,4 +4420,100 @@ mod tests { assert_eq!(result, "custom-scheme://panel.example.com/path?username=new&password=new"); } + + #[test] + fn panel_api_probe_targets_preserve_provider_failover_context() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "tivione".into(), + urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + name: Arc::from("cdn-dev"), + url: "provider://tivione".to_string(), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); + + assert_eq!(targets.len(), 4); + let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; + assert_eq!(*action, "client_info"); + assert_eq!( + input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), + "tivione" + ); + assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); + assert!(input_source.url.contains("username=probe-user")); + assert!(input_source.url.contains("password=probe-pass")); + assert!(input_source.url.contains("action=client_info")); + } + + #[test] + fn panel_api_probe_targets_keep_plain_http_without_provider_context() { + let input = ConfigInput { + name: Arc::from("plain"), + url: "http://origin.example.test/some/path?ignored=1".to_string(), + ..ConfigInput::default() + }; + + let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); + + assert_eq!(targets.len(), 4); + let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; + assert_eq!(*action, "client_info"); + assert!(input_source.provider.is_none()); + assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); + assert!(input_source.url.contains("username=probe-user")); + assert!(input_source.url.contains("password=probe-pass")); + assert!(input_source.url.contains("action=client_info")); + } + + #[test] + fn user_api_account_info_preserves_provider_failover_context() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "tivione".into(), + urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + name: Arc::from("cdn-dev"), + url: "provider://tivione".to_string(), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") + .expect("expected provider account_info input source"); + + assert_eq!( + input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), + "tivione" + ); + assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); + assert!(input_source.url.contains("username=root-user")); + assert!(input_source.url.contains("password=root-pass")); + assert!(input_source.url.contains("action=account_info")); + } + + #[test] + fn user_api_account_info_keeps_plain_http_without_provider_context() { + let input = ConfigInput { + name: Arc::from("plain"), + url: "http://origin.example.test/some/path?ignored=1".to_string(), + ..ConfigInput::default() + }; + + let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") + .expect("expected plain account_info input source"); + + assert!(input_source.provider.is_none()); + assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); + assert!(input_source.url.contains("username=root-user")); + assert!(input_source.url.contains("password=root-pass")); + assert!(input_source.url.contains("action=account_info")); + } } diff --git a/backend/src/api/setup_api.rs b/backend/src/api/setup_api.rs index 84625e9e2..8da941761 100644 --- a/backend/src/api/setup_api.rs +++ b/backend/src/api/setup_api.rs @@ -25,7 +25,8 @@ use shared::{ ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigApiDto, ConfigDto, ConfigPaths, PatternTemplate, SourcesConfigDto, TemplateDefinitionDto, TokenResponse, WebAuthConfigDto, WebUiConfigDto, TOKEN_NO_AUTH, }, - utils::{default_kick_secs, hex_encode, DEFAULT_PORT, DEFAULT_STORAGE_DIR, USER_FILE}, + utils::{hex_encode}, + defaults::{default_kick_secs, DEFAULT_PORT, DEFAULT_STORAGE_DIR, USER_FILE, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH}, }; use std::{ collections::HashSet, @@ -39,7 +40,6 @@ use std::{ }; use tokio::sync::{oneshot, Mutex, RwLock}; use tower_http::{compression::predicate::{DefaultPredicate, Predicate}, services::ServeDir}; -use shared::utils::DEFAULT_CUSTOM_STREAM_RESPONSE_PATH; const DEFAULT_SETUP_HOST: &str = "0.0.0.0"; diff --git a/backend/src/api/sys_usage.rs b/backend/src/api/sys_usage.rs index 6a4714c51..a54ac8c46 100644 --- a/backend/src/api/sys_usage.rs +++ b/backend/src/api/sys_usage.rs @@ -1,6 +1,7 @@ use crate::api::model::AppState; use crate::messaging::send_message as send_messaging; use crate::model::{DiskAlertConfig, MessageContent}; +#[cfg(target_os = "linux")] use crate::utils::parse_ascii_u64_bytes; use shared::model::{DiskAlert, DiskAlertLevel, MsgKind, SystemInfo}; use std::{ @@ -109,9 +110,9 @@ impl DiskProbe { if rc != 0 { return (0, 0); } - let bsize = stat.f_frsize; - let total = stat.f_blocks.saturating_mul(bsize); - let free = stat.f_bavail.saturating_mul(bsize); + let bsize = stat.f_frsize as u64; + let total = (stat.f_blocks as u64).saturating_mul(bsize); + let free = (stat.f_bavail as u64).saturating_mul(bsize); (total, free) } #[cfg(windows)] @@ -273,6 +274,7 @@ impl NetTracker { enum SystemUsageSampler { Platform(Box), + #[cfg(target_os = "linux")] Unavailable, #[cfg(not(target_os = "linux"))] Fallback(Box), @@ -297,6 +299,7 @@ impl SystemUsageSampler { fn sample(&mut self) -> Option { match self { Self::Platform(sampler) => sampler.sample(), + #[cfg(target_os = "linux")] Self::Unavailable => None, #[cfg(not(target_os = "linux"))] Self::Fallback(sampler) => sampler.sample(), @@ -335,14 +338,14 @@ impl FallbackSampler { self.networks.refresh(true); let (rx_bytes, tx_bytes) = sum_sysinfo_network_bytes(&self.networks); - let (net_rx_bytes_per_sec, net_tx_bytes_per_sec) = self.net_tracker.sample(rx_bytes, tx_bytes); + let (rx_bytes_per_sec, tx_bytes_per_sec) = self.net_tracker.sample(rx_bytes, tx_bytes); self.inner.processes().get(&self.pid).map(|proc| SystemInfo { cpu_usage: proc.cpu_usage(), memory_usage: proc.memory(), memory_total: self.inner.total_memory(), - net_rx_bytes_per_sec, - net_tx_bytes_per_sec, + net_rx_bytes_per_sec: rx_bytes_per_sec, + net_tx_bytes_per_sec: tx_bytes_per_sec, disk_total_bytes: 0, disk_free_bytes: 0, }) @@ -626,7 +629,7 @@ mod platform { #[cfg(target_os = "macos")] mod platform { use super::{CpuTracker, DiskProbe, SystemInfo}; - use libc::{c_char, c_int, c_void, getrusage, gettimeofday, rusage, sysctlbyname, timeval, RUSAGE_SELF}; + use libc::{c_int, c_void, getrusage, rusage, sysctlbyname, timeval, RUSAGE_SELF}; use std::{ ffi::CString, mem::{size_of, zeroed}, @@ -715,9 +718,9 @@ mod platform { let mut size = size_of::(); let rc = unsafe { sysctlbyname( - name.as_ptr() as *const c_char, - (&mut value as *mut u64).cast::(), - &mut size, + name.as_ptr().cast(), + (&raw mut value).cast::(), + &raw mut size, std::ptr::null_mut(), 0, ) @@ -732,8 +735,8 @@ mod platform { task_info( mach_task_self(), MACH_TASK_BASIC_INFO, - (&mut info as *mut MachTaskBasicInfo).cast::(), - &mut count, + (&raw mut info).cast::(), + &raw mut count, ) }; (rc == KERN_SUCCESS).then_some(info.resident_size) @@ -741,7 +744,7 @@ mod platform { fn query_process_cpu_time_secs() -> Option { let mut usage = unsafe { zeroed::() }; - let rc = unsafe { getrusage(RUSAGE_SELF, &mut usage) }; + let rc = unsafe { getrusage(RUSAGE_SELF, &raw mut usage) }; if rc != 0 { return None; } @@ -751,7 +754,8 @@ mod platform { Some(user_secs + system_secs) } - fn timeval_to_secs(tv: timeval) -> f64 { tv.tv_sec as f64 + (tv.tv_usec as f64 / 1_000_000.0) } + #[allow(clippy::cast_precision_loss)] + fn timeval_to_secs(tv: timeval) -> f64 { tv.tv_sec as f64 + (f64::from(tv.tv_usec) / 1_000_000.0) } } #[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index c8b743031..c3b439b9e 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -1,5 +1,8 @@ use log::{debug, error, warn}; -use shared::utils::{clean_playlist_title, TMDB_API_KEY}; +use shared::{ + utils::{clean_playlist_title}, + defaults::{TMDB_API_KEY}, +}; use crate::library::metadata::{MediaMetadata, MetadataSource, MovieMetadata, SeriesMetadata}; use crate::library::scanner::ScannedMediaFile; use crate::library::tmdb_client::TmdbClient; diff --git a/backend/src/library/processor.rs b/backend/src/library/processor.rs index da300ce18..7f47e6092 100644 --- a/backend/src/library/processor.rs +++ b/backend/src/library/processor.rs @@ -73,10 +73,10 @@ pub fn resolve_metadata_storage_path( storage_dir: &str, ) -> PathBuf { let configured_path = metadata_update_config.map_or_else( - || PathBuf::from(shared::utils::default_metadata_path()), + || PathBuf::from(shared::defaults::default_metadata_path()), |c| { if c.cache_path.is_empty() { - PathBuf::from(shared::utils::default_metadata_path()) + PathBuf::from(shared::defaults::default_metadata_path()) } else { PathBuf::from(c.cache_path.clone()) } diff --git a/backend/src/main.rs b/backend/src/main.rs index 1d5b34c3a..57f3e9196 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -29,8 +29,8 @@ use chrono::{DateTime, Utc}; use clap::Parser; use log::{error, info, warn}; use shared::{ + defaults::{CONFIG_FILE, CONFIG_PATH, DEFAULT_STORAGE_DIR, SOURCE_FILE}, model::ConfigPaths, - utils::{CONFIG_FILE, CONFIG_PATH, DEFAULT_STORAGE_DIR, SOURCE_FILE}, }; use std::{ fs::File, diff --git a/backend/src/model/config/app.rs b/backend/src/model/config/app.rs index f51bbb407..029e59b80 100644 --- a/backend/src/model/config/app.rs +++ b/backend/src/model/config/app.rs @@ -10,9 +10,10 @@ use log::{error, warn}; use rand::Rng; use shared::error::TuliproxError; use shared::model::{ConfigPaths, GeoIpUnavailablePolicy}; -use shared::utils::{ - CHANNEL_UNAVAILABLE, LOW_PRIORITY_PREEMPTED, PANEL_API_PROVISIONING, PROVIDER_CONNECTIONS_EXHAUSTED, - USER_ACCOUNT_EXPIRED, USER_CONNECTIONS_EXHAUSTED, +use shared::defaults::{ + CHANNEL_UNAVAILABLE, HLS_SESSION_OR_LEASE_EXPIRED, LOW_PRIORITY_PREEMPTED, PANEL_API_PROVISIONING, + PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT, PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX, + PROVIDER_CONNECTIONS_EXHAUSTED, USER_ACCOUNT_EXPIRED, USER_CONNECTIONS_EXHAUSTED, }; use std::borrow::Cow; use std::collections::HashSet; @@ -246,7 +247,7 @@ impl AppConfig { .api_proxy .load() .as_ref() - .map_or(shared::utils::default_auth_error_status(), |api_proxy| api_proxy.auth_error_status); + .map_or(shared::defaults::default_auth_error_status(), |api_proxy| api_proxy.auth_error_status); axum::http::StatusCode::from_u16(status).unwrap_or(axum::http::StatusCode::FORBIDDEN) } @@ -474,6 +475,13 @@ impl AppConfig { .or_else(|| provider_connections_exhausted.clone()); let user_account_expired = load_and_set_file(&path.join(USER_ACCOUNT_EXPIRED)); let panel_api_provisioning = load_and_set_file(&path.join(PANEL_API_PROVISIONING)); + let hls_session_or_lease_expired = load_and_set_file(&path.join(HLS_SESSION_OR_LEASE_EXPIRED)); + let panel_api_provisioning_hls_segments = (0..PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT) + .filter_map(|index| { + let filename = format!("{PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX}{index:03}.ts"); + load_and_set_file(&path.join(filename)) + }) + .collect(); self.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { channel_unavailable, user_connections_exhausted, @@ -481,6 +489,8 @@ impl AppConfig { low_priority_preempted, user_account_expired, panel_api_provisioning, + hls_session_or_lease_expired, + panel_api_provisioning_hls_segments, }))); } } diff --git a/backend/src/model/config/base.rs b/backend/src/model/config/base.rs index 53d16406d..9cb77e1a2 100644 --- a/backend/src/model/config/base.rs +++ b/backend/src/model/config/base.rs @@ -8,7 +8,8 @@ use log::{error, info}; use path_clean::PathClean; use shared::error::TuliproxError; use shared::model::{ConfigDto, GeoIpUnavailablePolicy, HdHomeRunDeviceOverview}; -use shared::utils::{default_grace_period_millis, default_grace_period_timeout_secs, set_sanitize_sensitive_info, DEFAULT_BACKUP_DIR, DEFAULT_CACHE_DIR, DEFAULT_DOWNLOAD_DIR, DEFAULT_STORAGE_DIR, DEFAULT_STORAGE_TEMP_DIR, DEFAULT_USER_CONFIG_DIR}; +use shared::utils::{ set_sanitize_sensitive_info}; +use shared::defaults::{default_grace_period_millis, default_grace_period_timeout_secs, DEFAULT_BACKUP_DIR, DEFAULT_CACHE_DIR, DEFAULT_DOWNLOAD_DIR, DEFAULT_STORAGE_DIR, DEFAULT_STORAGE_TEMP_DIR, DEFAULT_USER_CONFIG_DIR}; use std::borrow::Cow; use std::path::{Path, PathBuf}; use crate::utils::get_default_path_for_home; diff --git a/backend/src/model/config/cache.rs b/backend/src/model/config/cache.rs index c80741d0f..b1af3bf4b 100644 --- a/backend/src/model/config/cache.rs +++ b/backend/src/model/config/cache.rs @@ -1,5 +1,4 @@ -use shared::model::CacheConfigDto; -use shared::utils::parse_size_base_2; +use shared::model::{ByteSize, CacheConfigDto}; use crate::model::macros; #[derive(Debug, Clone)] @@ -7,7 +6,7 @@ pub struct CacheConfig { pub enabled: bool, pub directory: String, pub size: usize, - pub size_str: Option, + pub size_str: Option, } macros::from_impl!(CacheConfig); @@ -39,7 +38,7 @@ fn get_size(dto: &CacheConfigDto) -> usize { match dto.size.as_ref() { None => return 1024, Some(val) => { - if let Ok(size) = parse_size_base_2(val) { + if let Ok(size) = val.parse_bytes() { if let Ok(value) = usize::try_from(size) { return value; } diff --git a/backend/src/model/config/epg_smart_match.rs b/backend/src/model/config/epg_smart_match.rs index cb493c680..0ee789119 100644 --- a/backend/src/model/config/epg_smart_match.rs +++ b/backend/src/model/config/epg_smart_match.rs @@ -1,7 +1,8 @@ use std::sync::Arc; use regex::Regex; use shared::model::{EpgNamePrefix, EpgSmartMatchConfigDto}; -use shared::utils::{default_epg_name_prefix_separator, default_epg_strip, CONSTANTS}; +use shared::utils::{CONSTANTS}; +use shared::defaults::{default_epg_name_prefix_separator, default_epg_strip}; use crate::model::macros; #[derive(Debug, Clone)] diff --git a/backend/src/model/config/library.rs b/backend/src/model/config/library.rs index 63dc34dcc..0ca27347f 100644 --- a/backend/src/model/config/library.rs +++ b/backend/src/model/config/library.rs @@ -1,7 +1,7 @@ use crate::model::macros; use shared::error::TuliproxError; use shared::model::{ConfigDto, LibraryConfigDto, LibraryContentType, LibraryMetadataFormat}; -use shared::utils::DEFAULT_STORAGE_DIR; +use shared::defaults::DEFAULT_STORAGE_DIR; use shared::utils::Internable; use std::path::PathBuf; use std::sync::Arc; diff --git a/backend/src/model/config/log.rs b/backend/src/model/config/log.rs index 6d07fe52d..f3dbbe3eb 100644 --- a/backend/src/model/config/log.rs +++ b/backend/src/model/config/log.rs @@ -1,12 +1,10 @@ use shared::model::{LogConfigDto, RuntimeConfigReportFormat}; -use shared::utils::default_as_true; +use shared::defaults::default_as_true; use crate::model::macros; + // We need serde for these structs to read them during // start from the yaml file without reading the whole config. // - - - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] #[serde(deny_unknown_fields)] pub struct LogConfig { diff --git a/backend/src/model/config/metadata_update.rs b/backend/src/model/config/metadata_update.rs index 64ac8e374..018f8cc2c 100644 --- a/backend/src/model/config/metadata_update.rs +++ b/backend/src/model/config/metadata_update.rs @@ -1,8 +1,10 @@ use crate::model::macros; use shared::model::{ - FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, TmdbConfigDto, + ByteSize, FfprobeConfigDto, MetadataLogConfigDto, MetadataUpdateConfigDto, ProbeConfigDto, ResolveConfigDto, + TmdbConfigDto, }; -use shared::utils::{default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, default_tmdb_rate_limit_ms, parse_duration_seconds, parse_size_base_2}; +use shared::utils::{parse_duration_seconds}; +use shared::defaults::{default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_resolve_retry_backoff, default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, default_tmdb_rate_limit_ms}; #[derive(Debug, Clone)] pub struct MetadataUpdateConfig { @@ -63,11 +65,11 @@ pub struct FfprobeConfig { pub timeout: Option, pub analyze_duration: String, pub analyze_duration_micros: u64, - pub probe_size: String, + pub probe_size: ByteSize, pub probe_size_bytes: u64, pub live_analyze_duration: String, pub live_analyze_duration_micros: u64, - pub live_probe_size: String, + pub live_probe_size: ByteSize, pub live_probe_size_bytes: u64, } @@ -113,11 +115,12 @@ fn parse_duration_or_default(value: &str, default_value: &str, require_unit: boo .map_or(1, |v| v.max(1)) } -fn parse_size_or_default(value: &str, default_value: &str) -> u64 { - parse_size_base_2(value) +fn parse_size_or_default(value: &ByteSize, default_value: &ByteSize) -> u64 { + value + .parse_bytes() .ok() .map(|v| v.max(1)) - .or_else(|| parse_size_base_2(default_value).ok().map(|v| v.max(1))) + .or_else(|| default_value.parse_bytes().ok().map(|v| v.max(1))) .unwrap_or(1) } diff --git a/backend/src/model/config/reverse_proxy.rs b/backend/src/model/config/reverse_proxy.rs index 3438e7a8a..210f78d23 100644 --- a/backend/src/model/config/reverse_proxy.rs +++ b/backend/src/model/config/reverse_proxy.rs @@ -1,10 +1,16 @@ use crate::model::config::cache::CacheConfig; use crate::model::{macros, GeoIpConfig, QosAggregationConfig, RateLimitConfig, StreamConfig}; use regex::Regex; -use shared::model::{ResourceRetryConfigDto, ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, REGEX_CACHE}; -use shared::utils::{default_resource_retry_attempts, default_resource_retry_backoff_ms, default_resource_retry_backoff_multiplier, hex_to_u8_16, u8_16_to_hex}; +use shared::model::{ + HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsManifestRecoveryBurstConfigDto, + HlsSegmentRepairConfigDto, HlsSegmentRepairSizeIncreaseConfigDto, ResourceRetryConfigDto, ReverseProxyConfigDto, + ReverseProxyDisabledHeaderConfigDto, HlsStripMode, REGEX_CACHE, +}; +use shared::defaults::{default_resource_retry_attempts, default_resource_retry_backoff_ms, default_resource_retry_backoff_multiplier, }; +use shared::utils::{hex_to_u8_16, u8_16_to_hex}; use std::cmp::max; use std::sync::Arc; +use shared::defaults::HLS_CACHE_DIR_SUFFIX; #[derive(Debug, Clone)] pub struct ReverseProxyDisabledHeaderConfig { @@ -127,6 +133,223 @@ impl From<&ResourceRetryConfig> for ResourceRetryConfigDto { } } +// Runtime config structs hold the same enum types as their DTO counterparts +// (no mirror enums). `is_enabled`, `as_log_value`, and `plan` live on the +// shared enums (`shared::model::HlsCorruptSegmentWatchdogMode`, +// `shared::model::HlsManifestRecoveryBurstLevel`) so behavior stays in one +// place. The From impls below become trivial field-by-field clones. + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct StripConfig { + pub mode: HlsStripMode, + pub value: u64, +} + +impl From<&shared::model::HlsStripConfigDto> for StripConfig { + fn from(dto: &shared::model::HlsStripConfigDto) -> Self { + Self { mode: dto.mode, value: dto.value } + } +} + +impl From<&StripConfig> for shared::model::HlsStripConfigDto { + fn from(config: &StripConfig) -> Self { + Self { mode: config.mode, value: config.value } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsManifestRecoveryBurstConfig { + pub level: shared::model::HlsManifestRecoveryBurstLevel, +} + +impl Default for HlsManifestRecoveryBurstConfig { + fn default() -> Self { Self::from(&HlsManifestRecoveryBurstConfigDto::default()) } +} + +impl From<&HlsManifestRecoveryBurstConfigDto> for HlsManifestRecoveryBurstConfig { + fn from(dto: &HlsManifestRecoveryBurstConfigDto) -> Self { + Self { level: dto.level } + } +} + +impl From<&HlsManifestRecoveryBurstConfig> for HlsManifestRecoveryBurstConfigDto { + fn from(config: &HlsManifestRecoveryBurstConfig) -> Self { + Self { level: config.level } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsSegmentRepairSizeIncreaseConfig { + pub low_percent: u8, + pub medium_percent: u8, + pub high_percent: u8, +} + +impl Default for HlsSegmentRepairSizeIncreaseConfig { + fn default() -> Self { Self::from(&HlsSegmentRepairSizeIncreaseConfigDto::default()) } +} + +impl From<&HlsSegmentRepairSizeIncreaseConfigDto> for HlsSegmentRepairSizeIncreaseConfig { + fn from(dto: &HlsSegmentRepairSizeIncreaseConfigDto) -> Self { + Self { + low_percent: dto.low_percent, + medium_percent: dto.medium_percent, + high_percent: dto.high_percent, + } + } +} + +impl From<&HlsSegmentRepairSizeIncreaseConfig> for HlsSegmentRepairSizeIncreaseConfigDto { + fn from(config: &HlsSegmentRepairSizeIncreaseConfig) -> Self { + Self { + low_percent: config.low_percent, + medium_percent: config.medium_percent, + high_percent: config.high_percent, + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsSegmentRepairConfig { + pub max_level: shared::model::HlsSegmentRepairMode, + pub apply_to_first_segments: u8, + pub max_parallel_repairs: usize, + pub postprocess_timeout_ms: u64, + pub size_increase: HlsSegmentRepairSizeIncreaseConfig, + pub corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfig, +} + +impl Default for HlsSegmentRepairConfig { + fn default() -> Self { Self::from(&HlsSegmentRepairConfigDto::default()) } +} + +impl From<&HlsSegmentRepairConfigDto> for HlsSegmentRepairConfig { + fn from(dto: &HlsSegmentRepairConfigDto) -> Self { + Self { + max_level: dto.max_level, + apply_to_first_segments: dto.apply_to_first_segments, + max_parallel_repairs: dto.max_parallel_repairs, + postprocess_timeout_ms: dto.postprocess_timeout_ms, + size_increase: HlsSegmentRepairSizeIncreaseConfig::from(&dto.size_increase), + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfig::from(&dto.corrupt_segment_watchdog), + } + } +} + +impl From<&HlsSegmentRepairConfig> for HlsSegmentRepairConfigDto { + fn from(config: &HlsSegmentRepairConfig) -> Self { + Self { + max_level: config.max_level, + apply_to_first_segments: config.apply_to_first_segments, + max_parallel_repairs: config.max_parallel_repairs, + postprocess_timeout_ms: config.postprocess_timeout_ms, + size_increase: HlsSegmentRepairSizeIncreaseConfigDto::from(&config.size_increase), + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto::from(&config.corrupt_segment_watchdog), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsCorruptSegmentWatchdogConfig { + pub mode: shared::model::HlsCorruptSegmentWatchdogMode, + pub max_parallel_jobs: usize, +} + +impl Default for HlsCorruptSegmentWatchdogConfig { + fn default() -> Self { Self::from(&HlsCorruptSegmentWatchdogConfigDto::default()) } +} + +impl From<&HlsCorruptSegmentWatchdogConfigDto> for HlsCorruptSegmentWatchdogConfig { + fn from(dto: &HlsCorruptSegmentWatchdogConfigDto) -> Self { + Self { + mode: dto.mode, + max_parallel_jobs: dto.max_parallel_jobs, + } + } +} + +impl From<&HlsCorruptSegmentWatchdogConfig> for HlsCorruptSegmentWatchdogConfigDto { + fn from(config: &HlsCorruptSegmentWatchdogConfig) -> Self { + Self { + mode: config.mode, + max_parallel_jobs: config.max_parallel_jobs, + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsCacheConfig { + pub cache_path: String, + pub strip: StripConfig, + pub cache_duration: u64, + pub cache_bytes: u64, + pub cache_bytes_str: String, + pub cache_bytes_per_session: u64, + pub cache_bytes_per_session_str: String, + pub max_segments_prefetch: usize, + pub max_concurrent_segment_fetches_per_session: usize, + pub max_concurrent_segment_fetches_global: usize, + pub origin_manifest_timeout_ms: u64, + pub origin_segment_timeout_ms: u64, + pub session_idle_timeout: u64, + pub manifest_recovery_burst: HlsManifestRecoveryBurstConfig, + pub segment_repair: HlsSegmentRepairConfig, +} + +fn parse_hls_byte_size_or_default(value: &shared::model::ByteSize, default_value: &str) -> u64 { + value.parse_bytes().unwrap_or_else(|_| { + shared::model::ByteSize::new(default_value) + .parse_bytes() + .unwrap_or_default() + }) +} + +pub fn default_hls_cache_path() -> String { + std::env::temp_dir().join(HLS_CACHE_DIR_SUFFIX).to_string_lossy().into_owned() +} + +impl From<&HlsCacheConfigDto> for HlsCacheConfig { + fn from(dto: &HlsCacheConfigDto) -> Self { + Self { + cache_path: dto.cache_path.as_ref().map_or_else(default_hls_cache_path, |path: &String| Clone::clone(path)), + strip: StripConfig::from(&dto.strip), + cache_duration: dto.cache_duration, + cache_bytes: parse_hls_byte_size_or_default(&dto.cache_bytes, "10GB"), + cache_bytes_str: dto.cache_bytes.as_str().to_string(), + cache_bytes_per_session: parse_hls_byte_size_or_default(&dto.cache_bytes_per_session, "512MB"), + cache_bytes_per_session_str: dto.cache_bytes_per_session.as_str().to_string(), + max_segments_prefetch: dto.max_segments_prefetch, + max_concurrent_segment_fetches_per_session: dto.max_concurrent_segment_fetches_per_session, + max_concurrent_segment_fetches_global: dto.max_concurrent_segment_fetches_global, + origin_manifest_timeout_ms: dto.origin_manifest_timeout_ms, + origin_segment_timeout_ms: dto.origin_segment_timeout_ms, + session_idle_timeout: dto.session_idle_timeout, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::from(&dto.manifest_recovery_burst), + segment_repair: HlsSegmentRepairConfig::from(&dto.segment_repair), + } + } +} + +impl From<&HlsCacheConfig> for HlsCacheConfigDto { + fn from(config: &HlsCacheConfig) -> Self { + Self { + cache_path: Some(config.cache_path.clone()), + strip: shared::model::HlsStripConfigDto::from(&config.strip), + cache_duration: config.cache_duration, + cache_bytes: shared::model::ByteSize::new(config.cache_bytes_str.clone()), + cache_bytes_per_session: shared::model::ByteSize::new(config.cache_bytes_per_session_str.clone()), + max_segments_prefetch: config.max_segments_prefetch, + max_concurrent_segment_fetches_per_session: config.max_concurrent_segment_fetches_per_session, + max_concurrent_segment_fetches_global: config.max_concurrent_segment_fetches_global, + origin_manifest_timeout_ms: config.origin_manifest_timeout_ms, + origin_segment_timeout_ms: config.origin_segment_timeout_ms, + session_idle_timeout: config.session_idle_timeout, + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto::from(&config.manifest_recovery_burst), + segment_repair: HlsSegmentRepairConfigDto::from(&config.segment_repair), + } + } +} + #[derive(Debug, Clone)] pub struct ReverseProxyConfig { pub resource_rewrite_disabled: bool, @@ -139,6 +362,7 @@ pub struct ReverseProxyConfig { pub geoip: Option, pub stream_history: Option, pub qos_aggregation: Option, + pub hls_cache: Option, } macros::from_impl!(ReverseProxyConfig); @@ -164,6 +388,7 @@ impl From<&ReverseProxyConfigDto> for ReverseProxyConfig { geoip: dto.geoip.as_ref().map(Into::into), stream_history: dto.stream_history.as_ref().map(Into::into), qos_aggregation: dto.qos_aggregation.as_ref().map(Into::into), + hls_cache: dto.hls_cache.as_ref().map(Into::into), } } } @@ -186,14 +411,18 @@ impl From<&ReverseProxyConfig> for ReverseProxyConfigDto { geoip: instance.geoip.as_ref().map(Into::into), stream_history: instance.stream_history.as_ref().map(Into::into), qos_aggregation: instance.qos_aggregation.as_ref().map(Into::into), + hls_cache: instance.hls_cache.as_ref().map(Into::into), } } } #[cfg(test)] mod tests { - use super::ReverseProxyConfig; - use shared::model::{QosAggregationConfigDto, ReverseProxyConfigDto, StreamHistoryConfigDto}; + use super::{ + HlsCacheConfig, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, + ReverseProxyConfig, + }; + use shared::model::{ByteSize, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, QosAggregationConfigDto, ReverseProxyConfigDto, StreamHistoryConfigDto}; #[test] fn reverse_proxy_config_preserves_nested_stream_history() { @@ -237,4 +466,87 @@ mod tests { assert!(qos.enabled); assert_eq!(qos.interval_secs, 300); } + + #[test] + fn manifest_recovery_beast_burst_uses_two_lanes_per_aggressive_slot() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + + assert_eq!(plan.slots, 6); + assert_eq!(plan.lanes_per_slot, 2); + assert_eq!(plan.total_candidates(), 12); + assert_eq!(plan.slot_for_candidate(0), 0); + assert_eq!(plan.slot_for_candidate(1), 0); + assert_eq!(plan.slot_for_candidate(2), 1); + assert_eq!(plan.slot_for_candidate(11), 5); + } + + #[test] + fn reverse_proxy_config_preserves_default_hls_cache_settings() { + let dto = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + }; + + let config = ReverseProxyConfig::from(&dto); + let hls = config.hls_cache.expect("hls_cache should exist"); + + assert_eq!( + hls, + HlsCacheConfig { + cache_path: "/tmp/tuliprox/cache/hls".to_string(), + strip: super::StripConfig { + mode: HlsStripMode::Segments, + value: 0, + }, + cache_duration: 300, + cache_bytes: 10_737_418_240, // 10 * 1024^3 (was 10 * 1e9 under SI decimal before consolidation) + cache_bytes_str: "10GB".to_string(), + cache_bytes_per_session: 536_870_912, // 512 * 1024^2 (was 512 * 1e6 under SI decimal before consolidation) + cache_bytes_per_session_str: "512MB".to_string(), + max_segments_prefetch: 6, + max_concurrent_segment_fetches_per_session: 2, + max_concurrent_segment_fetches_global: 64, + origin_manifest_timeout_ms: 3_000, + origin_segment_timeout_ms: 10_000, + session_idle_timeout: 300, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + segment_repair: HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + }, + } + ); + } + + #[test] + fn hls_cache_runtime_config_parses_byte_sizes() { + let dto = HlsCacheConfigDto { + cache_bytes: ByteSize::new("1GiB"), + cache_bytes_per_session: ByteSize::new("512MB"), + ..Default::default() + }; + + let config = HlsCacheConfig::from(&dto); + + assert_eq!(config.cache_bytes, 1_073_741_824); + assert_eq!(config.cache_bytes_per_session, 536_870_912); // 512 * 1024^2 (binary since parse_size_base_2 consolidation) + } + + #[test] + fn hls_cache_runtime_config_roundtrips_human_readable_sizes() { + let dto = HlsCacheConfigDto { + cache_bytes: ByteSize::new("1GiB"), + cache_bytes_per_session: ByteSize::new("512MB"), + ..Default::default() + }; + + let config = HlsCacheConfig::from(&dto); + let roundtrip = HlsCacheConfigDto::from(&config); + + assert_eq!(roundtrip.cache_bytes.as_str(), "1GiB"); + assert_eq!(roundtrip.cache_bytes_per_session.as_str(), "512MB"); + } } diff --git a/backend/src/model/config/source.rs b/backend/src/model/config/source.rs index c04315fe3..41911c740 100644 --- a/backend/src/model/config/source.rs +++ b/backend/src/model/config/source.rs @@ -107,7 +107,7 @@ impl ProviderDnsCache { let mut guard = self.by_host.write(); let entry = guard.entry(host.to_ascii_lowercase()).or_default(); // `% len` guards against a stale index when the override list changed length. - let idx = entry.rr_index.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; + let idx = entry.rr_index.try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; Some(ips[idx]) } @@ -118,7 +118,7 @@ impl ProviderDnsCache { return None; } let len = entry.ips.len(); - let idx = entry.rr_index.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; + let idx = entry.rr_index.try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some((i + 1) % len)).unwrap_or_else(|i| i) % len; Some(entry.ips[idx]) } @@ -141,7 +141,7 @@ impl ProviderDnsCache { if new_len == 0 { entry.rr_index.store(0, Ordering::Relaxed); } else { - entry.rr_index.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some(i % new_len)).ok(); + entry.rr_index.try_update(Ordering::Relaxed, Ordering::Relaxed, |i| Some(i % new_len)).ok(); } entry.last_ok = Some(SystemTime::now()); entry.last_err = None; @@ -358,7 +358,7 @@ impl ConfigProvider { } let previous = self.current_url_index - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { let next = (current + 1) % len; // If we've cycled back to start, we've tried all URLs. (next != start_index).then_some(next) diff --git a/backend/src/model/config/stream.rs b/backend/src/model/config/stream.rs index 803870609..35a82aaf9 100644 --- a/backend/src/model/config/stream.rs +++ b/backend/src/model/config/stream.rs @@ -164,4 +164,6 @@ pub struct CustomStreamResponse { pub low_priority_preempted: Option, // stream was preempted by a higher-priority user pub user_account_expired: Option, pub panel_api_provisioning: Option, + pub hls_session_or_lease_expired: Option, + pub panel_api_provisioning_hls_segments: Vec, } diff --git a/backend/src/model/config/stream_history.rs b/backend/src/model/config/stream_history.rs index 861b52441..858cd7b8f 100644 --- a/backend/src/model/config/stream_history.rs +++ b/backend/src/model/config/stream_history.rs @@ -2,7 +2,7 @@ use crate::model::macros; use shared::model::{ StreamHistoryConfigDto, }; -use shared::utils::{default_stream_history_batch_size, default_stream_history_retention_days}; +use shared::defaults::{default_stream_history_batch_size, default_stream_history_retention_days}; #[derive(Debug, Clone)] pub struct StreamHistoryConfig { diff --git a/backend/src/model/config/video_download.rs b/backend/src/model/config/video_download.rs index 3d08b26d2..7a1efbab3 100644 --- a/backend/src/model/config/video_download.rs +++ b/backend/src/model/config/video_download.rs @@ -1,7 +1,7 @@ use crate::model::macros; use regex::Regex; use shared::model::{VideoConfigDto, VideoDownloadConfigDto}; -use shared::utils::DEFAULT_DOWNLOAD_DIR; +use shared::defaults::DEFAULT_DOWNLOAD_DIR; use std::collections::HashMap; use std::sync::Arc; diff --git a/backend/src/model/config/web_auth.rs b/backend/src/model/config/web_auth.rs index c4464f496..552ea755b 100644 --- a/backend/src/model/config/web_auth.rs +++ b/backend/src/model/config/web_auth.rs @@ -4,6 +4,7 @@ use std::io::{BufRead, ErrorKind}; use std::path::{Path, PathBuf}; use log::{debug, error, trace, warn}; +use shared::defaults::{is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path}; use shared::error::TuliproxError; use shared::model::permission::{permission_from_name, PermissionSet, PERM_ALL}; use shared::model::WebAuthConfigDto; @@ -78,8 +79,8 @@ impl From<&WebAuthConfig> for WebAuthConfigDto { impl WebAuthConfig { pub fn prepare(&mut self, config_path: &str) -> Result<(), TuliproxError> { - let has_custom_userfile = !utils::is_blank_or_default_user_file_path(&self.userfile); - let has_custom_groupfile = !utils::is_blank_or_default_user_group_file_path(&self.groupfile); + let has_custom_userfile = !is_blank_or_default_user_file_path(&self.userfile); + let has_custom_groupfile = !is_blank_or_default_user_group_file_path(&self.groupfile); let userfile_name = if has_custom_userfile { self.userfile.as_ref().map_or_else(String::new, std::borrow::ToOwned::to_owned) } else { diff --git a/backend/src/model/config/web_ui.rs b/backend/src/model/config/web_ui.rs index e4af3994b..06d75cc66 100644 --- a/backend/src/model/config/web_ui.rs +++ b/backend/src/model/config/web_ui.rs @@ -2,7 +2,7 @@ use crate::model::{macros, WebAuthConfig}; use shared::error::TuliproxError; use shared::model::view_type::ViewType; use shared::model::{ContentSecurityPolicyConfigDto, StreamInfoConfigDto, StreamInfoFields, StreamInfoFieldsSet, WebUiConfigDto}; -use shared::utils::default_kick_secs; +use shared::defaults::default_kick_secs; #[derive(Debug, Clone)] pub struct StreamInfoConfig { diff --git a/backend/src/processing/parser/hls.rs b/backend/src/processing/parser/hls.rs index 92363d8c5..3a09cc44b 100644 --- a/backend/src/processing/parser/hls.rs +++ b/backend/src/processing/parser/hls.rs @@ -1,10 +1,17 @@ use crate::model::ProxyUserCredentials; use shared::concat_string; -use shared::utils::{deobfuscate_text, extract_extension_from_url, obfuscate_text, CONSTANTS, HLS_PREFIX}; +use shared::{ + utils::{deobfuscate_text, extract_extension_from_url, obfuscate_text, CONSTANTS}, + defaults::{HLS_PREFIX} +}; use std::borrow::Cow; use std::str; use url::Url; +pub mod origin_manifest; +pub mod initial_strip; +pub mod transient_manifest; + const TOKEN_SEPARATOR: char = '\x1F'; const TOKEN_SEPARATOR_STR: &str = "\x1F"; @@ -148,7 +155,8 @@ mod test { get_hls_session_token_and_url_from_token, rewrite_hls, rewrite_hls_url, RewriteHlsProps, }; use rand::RngCore; - use shared::utils::{u32_to_base64, HLS_PREFIX}; + use shared::utils::{u32_to_base64}; + use shared::defaults::{HLS_PREFIX}; #[test] fn test_token_size() { diff --git a/backend/src/processing/parser/hls/initial_strip.rs b/backend/src/processing/parser/hls/initial_strip.rs new file mode 100644 index 000000000..75242760a --- /dev/null +++ b/backend/src/processing/parser/hls/initial_strip.rs @@ -0,0 +1,226 @@ +use super::transient_manifest::{ + configured_strip_segments, manifest_lines, media_segment_units, strip_mode_log_value, + MIN_HLS_INITIAL_VISIBLE_SEGMENTS, +}; +use crate::model::StripConfig; +use shared::model::HlsStripMode; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsInitialStripView { + pub body: String, + pub outcome: HlsInitialStripOutcome, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum HlsInitialStripOutcome { + Applied { + mode: &'static str, + configured: u64, + effective: usize, + visible_segments: usize, + }, + Skipped { + reason: HlsInitialStripSkipReason, + visible_segments: usize, + }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsInitialStripSkipReason { + StripDisabled, + NotEnoughSegments, +} + +impl HlsInitialStripSkipReason { + pub const fn as_log_reason(self) -> &'static str { + match self { + Self::StripDisabled => "strip-disabled", + Self::NotEnoughSegments => "not-enough-segments", + } + } +} + +pub fn materialize_initial_hls_strip_view(body: &str, strip: &StripConfig) -> HlsInitialStripView { + let lines = manifest_lines(body); + let units = media_segment_units(&lines); + let media_segment_count = units.len(); + if strip.value == 0 { + return HlsInitialStripView { + body: body.to_string(), + outcome: HlsInitialStripOutcome::Skipped { + reason: HlsInitialStripSkipReason::StripDisabled, + visible_segments: media_segment_count, + }, + }; + } + + let effective_strip_segments = effective_initial_hls_strip_segments(strip, &lines, &units); + if effective_strip_segments == 0 { + return HlsInitialStripView { + body: body.to_string(), + outcome: HlsInitialStripOutcome::Skipped { + reason: HlsInitialStripSkipReason::NotEnoughSegments, + visible_segments: media_segment_count, + }, + }; + } + + let visible_segments = media_segment_count.saturating_sub(effective_strip_segments); + let strip_start_unit = media_segment_count.saturating_sub(effective_strip_segments); + let strip_ranges = &units[strip_start_unit..]; + let mut stripped = String::with_capacity(body.len()); + for (index, line) in lines.iter().enumerate() { + if strip_ranges.iter().any(|unit| unit.contains(index)) { + continue; + } + stripped.push_str(line.line); + stripped.push_str(line.ending); + } + + HlsInitialStripView { + body: stripped, + outcome: HlsInitialStripOutcome::Applied { + mode: strip_mode_log_value(strip.mode), + configured: strip.value, + effective: effective_strip_segments, + visible_segments, + }, + } +} + +pub fn initial_hls_strip_segments_for_durations(strip: &StripConfig, segment_durations_ms: &[u64]) -> usize { + if strip.value == 0 { + return 0; + } + let configured_segments = match strip.mode { + HlsStripMode::Segments => usize::try_from(strip.value).unwrap_or(usize::MAX), + HlsStripMode::Seconds => configured_strip_segments_from_durations(strip.value, segment_durations_ms), + }; + let max_removable_segments = segment_durations_ms.len().saturating_sub(MIN_HLS_INITIAL_VISIBLE_SEGMENTS); + configured_segments.min(max_removable_segments) +} + +fn effective_initial_hls_strip_segments( + strip: &StripConfig, + lines: &[super::transient_manifest::ManifestLine<'_>], + units: &[super::transient_manifest::MediaSegmentUnit], +) -> usize { + let configured_segments = configured_strip_segments(strip, lines, units); + let max_removable_segments = units.len().saturating_sub(MIN_HLS_INITIAL_VISIBLE_SEGMENTS); + configured_segments.min(max_removable_segments) +} + +fn configured_strip_segments_from_durations(strip_seconds: u64, segment_durations_ms: &[u64]) -> usize { + let target_ms = strip_seconds.saturating_mul(1_000); + let mut accumulated_ms = 0_u64; + let mut strip_segments = 0_usize; + for duration_ms in segment_durations_ms.iter().rev() { + strip_segments = strip_segments.saturating_add(1); + accumulated_ms = accumulated_ms.saturating_add(*duration_ms); + if accumulated_ms >= target_ms { + break; + } + } + strip_segments +} + +#[cfg(test)] +mod tests { + use super::{materialize_initial_hls_strip_view, HlsInitialStripOutcome, HlsInitialStripSkipReason}; + use crate::model::StripConfig; +use shared::model::HlsStripMode; + use std::fmt::Write as _; + + fn strip_segments(value: u64) -> StripConfig { StripConfig { mode: HlsStripMode::Segments, value } } + + fn strip_seconds(value: u64) -> StripConfig { StripConfig { mode: HlsStripMode::Seconds, value } } + + fn manifest_with_segments(count: usize) -> String { + let mut body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n".to_string(); + for index in 0..count { + body.push_str("#EXTINF:10.0,\n"); + writeln!(&mut body, "seg{index}.ts").expect("write segment URI"); + } + body + } + + fn media_segment_count(body: &str) -> usize { + body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).count() + } + + #[test] + fn pending_strip_segments_keeps_visible_head_window() { + let view = materialize_initial_hls_strip_view(&manifest_with_segments(6), &strip_segments(3)); + + assert_eq!(media_segment_count(&view.body), 3); + assert!(matches!( + view.outcome, + HlsInitialStripOutcome::Applied { + mode: "segments", + configured: 3, + effective: 3, + visible_segments: 3, + } + )); + } + + #[test] + fn pending_strip_segments_never_keeps_less_than_three_segments() { + let four_segment_view = materialize_initial_hls_strip_view(&manifest_with_segments(4), &strip_segments(3)); + let three_segment_view = materialize_initial_hls_strip_view(&manifest_with_segments(3), &strip_segments(3)); + + assert_eq!(media_segment_count(&four_segment_view.body), 3); + assert_eq!(media_segment_count(&three_segment_view.body), 3); + assert!(matches!( + three_segment_view.outcome, + HlsInitialStripOutcome::Skipped { + reason: HlsInitialStripSkipReason::NotEnoughSegments, + visible_segments: 3, + } + )); + } + + #[test] + fn pending_strip_seconds_counts_tail_extinf_durations() { + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:10.0,\nseg100.ts\n#EXTINF:9.0,\nseg101.ts\n#EXTINF:9.0,\nseg102.ts\n#EXTINF:9.0,\nseg103.ts\n#EXTINF:9.0,\nseg104.ts\n#EXTINF:9.0,\nseg105.ts\n"; + + let view = materialize_initial_hls_strip_view(body, &strip_seconds(30)); + + assert_eq!(media_segment_count(&view.body), 3); + assert!(matches!( + view.outcome, + HlsInitialStripOutcome::Applied { + mode: "seconds", + configured: 30, + effective: 3, + visible_segments: 3, + } + )); + } + + #[test] + fn pending_strip_disabled_keeps_full_body() { + let view = materialize_initial_hls_strip_view(&manifest_with_segments(4), &strip_segments(0)); + + assert_eq!(media_segment_count(&view.body), 4); + assert!(matches!( + view.outcome, + HlsInitialStripOutcome::Skipped { + reason: HlsInitialStripSkipReason::StripDisabled, + visible_segments: 4, + } + )); + } + + #[test] + fn pending_strip_preserves_media_sequence_and_byterange_semantics() { + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:42\n#EXT-X-BYTERANGE:100@200\n#EXTINF:4.0,\nseg42.ts\n#EXT-X-BYTERANGE:100\n#EXTINF:4.0,\nseg43.ts\n#EXT-X-BYTERANGE:100\n#EXTINF:4.0,\nseg44.ts\n#EXT-X-BYTERANGE:100\n#EXTINF:4.0,\nseg45.ts\n"; + + let view = materialize_initial_hls_strip_view(body, &strip_segments(1)); + + assert!(view.body.contains("#EXT-X-MEDIA-SEQUENCE:42")); + assert!(view.body.contains("#EXT-X-BYTERANGE:100@200")); + assert!(view.body.contains("#EXT-X-BYTERANGE:100\n#EXTINF:4.0,\nseg43.ts")); + assert!(!view.body.contains("seg45.ts")); + } +} diff --git a/backend/src/processing/parser/hls/origin_manifest.rs b/backend/src/processing/parser/hls/origin_manifest.rs new file mode 100644 index 000000000..4f04dfa72 --- /dev/null +++ b/backend/src/processing/parser/hls/origin_manifest.rs @@ -0,0 +1,826 @@ +#![allow(dead_code)] + +use super::rewrite_hls_url; +use shared::utils::CONSTANTS; +use std::{borrow::Cow, collections::HashMap, fmt}; + +/// Result of parsing an origin media playlist for the live HLS cache proxy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginManifestParseOutcome { + Normal(ParsedOriginManifest), + TransientPassthrough { reason: OriginManifestTransientReason }, +} + +/// Reason why an origin manifest cannot enter the normal cache timeline. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginManifestTransientReason { + ExtXKey, + UnsupportedTag { tag: String }, + ParserUnsupportedFeature { feature: String }, +} + +/// Parsed normal-timeline view of a live HLS origin media playlist. +#[derive(Clone, PartialEq, Eq)] +pub struct ParsedOriginManifest { + pub origin_manifest_sequence: u64, + pub origin_manifest_segment_cnt: usize, + pub version: Option, + pub target_duration: Option, + pub discontinuity_sequence: Option, + pub independent_segments: bool, + /// Parsed MAP fetch references before proxy MAP ID assignment. + pub maps: Vec, + /// Parsed segment fetch references before proxy sequence mapping. + pub segments: Vec, +} + +/// Timing values parsed from HLS manifest attributes used for origin refresh debounce. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ParsedManifestTiming { + pub target_duration_ms: Option, + pub last_segment_duration_ms: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParsedManifestValiditySource { + ExtInfSum, + TargetDuration, + Fallback, +} + +/// Parsed validity window for a committed HLS manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParsedManifestValidity { + pub playlist_duration_ms: u64, + pub segment_count: usize, + pub source: ParsedManifestValiditySource, +} + +/// Lightweight timeline markers used to validate transient passthrough manifests +/// without applying normal timeline parsing or URI normalization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParsedOriginManifestTimeline { + pub origin_manifest_sequence: u64, + pub origin_manifest_segment_cnt: usize, +} + +impl ParsedOriginManifestTimeline { + pub fn origin_highwater(self) -> Option { + let segment_count = u64::try_from(self.origin_manifest_segment_cnt).ok()?; + if segment_count == 0 { + return None; + } + self.origin_manifest_sequence.checked_add(segment_count.saturating_sub(1)) + } +} + +impl fmt::Debug for ParsedOriginManifest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParsedOriginManifest") + .field("origin_manifest_sequence", &self.origin_manifest_sequence) + .field("origin_manifest_segment_cnt", &self.origin_manifest_segment_cnt) + .field("version", &self.version) + .field("target_duration", &self.target_duration) + .field("discontinuity_sequence", &self.discontinuity_sequence) + .field("independent_segments", &self.independent_segments) + .field("maps_len", &self.maps.len()) + .field("segments_len", &self.segments.len()) + .finish() + } +} + +/// Parsed origin segment metadata before proxy sequence mapping and rendering. +#[derive(Clone, PartialEq, Eq)] +pub struct ParsedOriginSegment { + pub origin_seq: u64, + pub duration_ms: u64, + /// Concrete absolute segment fetch URL resolved against the final manifest URL after redirects. + /// + /// This may intentionally contain a provider mirror, redirect target, or CDN host. It is request-local fetch + /// metadata, not HLS session identity, account binding, or provider-failover state. + pub resolved_origin_url: String, + pub discontinuity_before: bool, + pub program_date_time: Option, + pub daterange_tags_before: Vec, + pub origin_byte_range: Option, + pub map_ref: Option, +} + +impl fmt::Debug for ParsedOriginSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParsedOriginSegment") + .field("origin_seq", &self.origin_seq) + .field("duration_ms", &self.duration_ms) + .field("resolved_origin_url", &"") + .field("discontinuity_before", &self.discontinuity_before) + .field("program_date_time", &self.program_date_time) + .field("daterange_tags_before", &self.daterange_tags_before) + .field("origin_byte_range", &self.origin_byte_range) + .field("map_ref", &self.map_ref) + .finish() + } +} + +/// Parsed origin MAP metadata before proxy MAP ID assignment and rendering. +#[derive(Clone, PartialEq, Eq)] +pub struct ParsedOriginMap { + pub map_id: usize, + /// Concrete absolute MAP URI resolved against the final manifest URL after + /// redirects. It is a fetch URI, not a provider-failover identity. + pub resolved_origin_uri: String, + pub byte_range: Option, +} + +impl fmt::Debug for ParsedOriginMap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParsedOriginMap") + .field("map_id", &self.map_id) + .field("resolved_origin_uri", &"") + .field("byte_range", &self.byte_range) + .finish() + } +} + +/// Absolute origin byte range normalized from an `EXT-X-BYTERANGE` tag. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParsedByteRange { + pub length: u64, + pub offset: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PendingByteRange { + WithOffset(ParsedByteRange), + WithoutOffset(u64), +} + +pub fn parse_origin_media_manifest(body: &str, final_manifest_url: &str) -> OriginManifestParseOutcome { + match parse_origin_media_manifest_result(body, final_manifest_url) { + Ok(manifest) => OriginManifestParseOutcome::Normal(manifest), + Err(reason) => OriginManifestParseOutcome::TransientPassthrough { reason }, + } +} + +pub fn parse_origin_manifest_timeline( + body: &str, +) -> Result { + let mut seen_extm3u = false; + let mut media_sequence = None; + let mut pending_extinf = false; + let mut segment_count = 0_usize; + + for line in body.lines().map(str::trim).filter(|line| !line.is_empty()) { + if line.starts_with('#') { + let tag = tag_name(line); + match tag { + "#EXTM3U" => seen_extm3u = true, + "#EXT-X-MEDIA-SEQUENCE" => { + media_sequence = Some(parse_tag_value(line, tag)?.parse_numeric("invalid_media_sequence")?); + } + "#EXTINF" => pending_extinf = true, + _ => {} + } + continue; + } + + if !pending_extinf { + return Err(parser_feature("segment_uri_without_extinf")); + } + pending_extinf = false; + segment_count = segment_count.saturating_add(1); + } + + if !seen_extm3u { + return Err(parser_feature("missing_extm3u")); + } + if pending_extinf { + return Err(parser_feature("extinf_without_segment_uri")); + } + + Ok(ParsedOriginManifestTimeline { + origin_manifest_sequence: media_sequence.unwrap_or(0), + origin_manifest_segment_cnt: segment_count, + }) +} + +pub fn parse_manifest_timing(body: &str) -> ParsedManifestTiming { + let mut timing = ParsedManifestTiming::default(); + for line in body.lines().map(str::trim).filter(|line| !line.is_empty()) { + if let Some(value) = line.strip_prefix("#EXT-X-TARGETDURATION:") { + if let Ok(duration) = value.trim().parse::() { + timing.target_duration_ms = Some(duration.saturating_mul(1_000)); + } + } else if let Some(value) = line.strip_prefix("#EXTINF:").and_then(parse_extinf_duration_millis) { + timing.last_segment_duration_ms = Some(value); + } + } + timing +} + +pub fn parse_manifest_validity(body: &str) -> Option { + let mut target_duration_ms = None; + let mut pending_extinf_duration_ms = None; + let mut segment_count = 0_usize; + let mut extinf_sum_ms = 0_u64; + let mut extinf_duration_count = 0_usize; + + for line in body.lines().map(str::trim).filter(|line| !line.is_empty()) { + if let Some(value) = line.strip_prefix("#EXT-X-TARGETDURATION:") { + if let Ok(duration) = value.trim().parse::() { + target_duration_ms = Some(duration.saturating_mul(1_000)); + } + continue; + } + + if let Some(value) = line.strip_prefix("#EXTINF:").and_then(parse_extinf_duration_millis) { + pending_extinf_duration_ms = Some(value); + continue; + } + + if line.starts_with('#') { + continue; + } + + segment_count = segment_count.saturating_add(1); + if let Some(duration_ms) = pending_extinf_duration_ms.take() { + extinf_sum_ms = extinf_sum_ms.saturating_add(duration_ms); + extinf_duration_count = extinf_duration_count.saturating_add(1); + } + } + + if segment_count == 0 { + return None; + } + + if extinf_duration_count == segment_count && extinf_sum_ms > 0 { + return Some(ParsedManifestValidity { + playlist_duration_ms: extinf_sum_ms, + segment_count, + source: ParsedManifestValiditySource::ExtInfSum, + }); + } + + if let Some(target_duration_ms) = target_duration_ms { + return Some(ParsedManifestValidity { + playlist_duration_ms: target_duration_ms.saturating_mul(u64::try_from(segment_count).ok()?), + segment_count, + source: ParsedManifestValiditySource::TargetDuration, + }); + } + + Some(ParsedManifestValidity { + playlist_duration_ms: 15_000_u64.saturating_mul(u64::try_from(segment_count).ok()?), + segment_count, + source: ParsedManifestValiditySource::Fallback, + }) +} + +fn parse_extinf_duration_millis(value: &str) -> Option { + let duration = value.split(',').next()?.trim(); + let (seconds, fraction) = duration.split_once('.').unwrap_or((duration, "")); + let seconds = seconds.parse::().ok()?; + let millis = fraction.chars().take(3).try_fold((0_u64, 0_u32), |(value, digits), digit| { + digit.to_digit(10).map(|digit| (value.saturating_mul(10).saturating_add(u64::from(digit)), digits + 1)) + })?; + let millis = millis.0.saturating_mul(10_u64.saturating_pow(3_u32.saturating_sub(millis.1))); + Some(seconds.saturating_mul(1_000).saturating_add(millis)) +} + +fn parse_origin_media_manifest_result( + body: &str, + final_manifest_url: &str, +) -> Result { + let mut parser = OriginManifestParser::new(final_manifest_url); + for line in body.lines().map(str::trim).filter(|line| !line.is_empty()) { + parser.parse_line(line)?; + } + parser.finish() +} + +struct OriginManifestParser<'a> { + final_manifest_url: &'a str, + seen_extm3u: bool, + version: Option, + target_duration: Option, + media_sequence: Option, + discontinuity_sequence: Option, + independent_segments: bool, + pending_extinf_duration_ms: Option, + pending_discontinuity: bool, + pending_program_date_time: Option, + pending_daterange_tags: Vec, + pending_byte_range: Option, + current_map_ref: Option, + maps: Vec, + segments_without_seq: Vec, + next_byte_range_offset_by_uri: HashMap, +} + +impl<'a> OriginManifestParser<'a> { + fn new(final_manifest_url: &'a str) -> Self { + Self { + final_manifest_url, + seen_extm3u: false, + version: None, + target_duration: None, + media_sequence: None, + discontinuity_sequence: None, + independent_segments: false, + pending_extinf_duration_ms: None, + pending_discontinuity: false, + pending_program_date_time: None, + pending_daterange_tags: Vec::new(), + pending_byte_range: None, + current_map_ref: None, + maps: Vec::new(), + segments_without_seq: Vec::new(), + next_byte_range_offset_by_uri: HashMap::new(), + } + } + + fn parse_line(&mut self, line: &str) -> Result<(), OriginManifestTransientReason> { + if line.starts_with('#') { + return self.parse_tag_line(line); + } + self.parse_segment_uri(line) + } + + fn parse_tag_line(&mut self, line: &str) -> Result<(), OriginManifestTransientReason> { + if !line.starts_with("#EXT") { + return Ok(()); + } + let tag = tag_name(line); + if tag == "#EXT-X-KEY" { + return Err(OriginManifestTransientReason::ExtXKey); + } + if !is_allowed_normal_timeline_tag(tag) { + return Err(OriginManifestTransientReason::UnsupportedTag { tag: tag.to_string() }); + } + + match tag { + "#EXTM3U" => self.seen_extm3u = true, + "#EXT-X-VERSION" => self.version = Some(parse_tag_value(line, tag)?.parse_numeric("invalid_version")?), + "#EXT-X-TARGETDURATION" => { + self.target_duration = Some(parse_tag_value(line, tag)?.parse_numeric("invalid_target_duration")?); + } + "#EXT-X-MEDIA-SEQUENCE" => { + self.media_sequence = Some(parse_tag_value(line, tag)?.parse_numeric("invalid_media_sequence")?); + } + "#EXTINF" => self.pending_extinf_duration_ms = Some(parse_extinf_duration_ms(line)?), + "#EXT-X-DISCONTINUITY" => self.pending_discontinuity = true, + "#EXT-X-DISCONTINUITY-SEQUENCE" => { + self.discontinuity_sequence = + Some(parse_tag_value(line, tag)?.parse_numeric("invalid_discontinuity_sequence")?); + } + "#EXT-X-MAP" => self.current_map_ref = Some(self.parse_map(line)?), + "#EXT-X-BYTERANGE" => self.pending_byte_range = Some(parse_byterange_tag(line)?), + "#EXT-X-PROGRAM-DATE-TIME" => { + self.pending_program_date_time = Some(parse_tag_value(line, tag)?.to_string()); + } + "#EXT-X-DATERANGE" => self.pending_daterange_tags.push(line.to_string()), + "#EXT-X-INDEPENDENT-SEGMENTS" => self.independent_segments = true, + _ => {} + } + Ok(()) + } + + fn parse_map(&mut self, line: &str) -> Result { + let Some(captures) = CONSTANTS.re_hls_uri.captures(line) else { + return Err(parser_feature("map_without_uri")); + }; + let Some(uri) = captures.get(1).map(|m| m.as_str()) else { + return Err(parser_feature("map_without_uri")); + }; + + let resolved_origin_uri = resolve_uri(self.final_manifest_url, uri); + let byte_range = parse_attribute(line, "BYTERANGE").map(parse_map_byterange).transpose()?; + let map_id = self.maps.len(); + self.maps.push(ParsedOriginMap { map_id, resolved_origin_uri, byte_range }); + Ok(map_id) + } + + fn parse_segment_uri(&mut self, line: &str) -> Result<(), OriginManifestTransientReason> { + let Some(duration_ms) = self.pending_extinf_duration_ms.take() else { + return Err(parser_feature("segment_uri_without_extinf")); + }; + let resolved_origin_url = resolve_uri(self.final_manifest_url, line); + let origin_byte_range = self.resolve_pending_byte_range(&resolved_origin_url)?; + self.segments_without_seq.push(ParsedOriginSegment { + origin_seq: 0, + duration_ms, + resolved_origin_url, + discontinuity_before: std::mem::take(&mut self.pending_discontinuity), + program_date_time: self.pending_program_date_time.take(), + daterange_tags_before: std::mem::take(&mut self.pending_daterange_tags), + origin_byte_range, + map_ref: self.current_map_ref, + }); + Ok(()) + } + + fn resolve_pending_byte_range( + &mut self, + resolved_origin_url: &str, + ) -> Result, OriginManifestTransientReason> { + let Some(pending) = self.pending_byte_range.take() else { + return Ok(None); + }; + let byte_range = match pending { + PendingByteRange::WithOffset(byte_range) => byte_range, + PendingByteRange::WithoutOffset(length) => { + let Some(offset) = self.next_byte_range_offset_by_uri.get(resolved_origin_url).copied() else { + return Err(parser_feature("byterange_without_resolvable_offset")); + }; + ParsedByteRange { length, offset } + } + }; + self.next_byte_range_offset_by_uri.insert( + resolved_origin_url.to_string(), + byte_range + .offset + .checked_add(byte_range.length) + .ok_or_else(|| parser_feature("byterange_offset_overflow"))?, + ); + Ok(Some(byte_range)) + } + + fn finish(mut self) -> Result { + if !self.seen_extm3u { + return Err(parser_feature("missing_extm3u")); + } + if self.pending_extinf_duration_ms.is_some() { + return Err(parser_feature("extinf_without_segment_uri")); + } + + let origin_manifest_sequence = self.media_sequence.unwrap_or(0); + for (idx, segment) in self.segments_without_seq.iter_mut().enumerate() { + segment.origin_seq = origin_manifest_sequence + .checked_add(u64::try_from(idx).map_err(|_| parser_feature("too_many_segments"))?) + .ok_or_else(|| parser_feature("origin_sequence_overflow"))?; + } + + Ok(ParsedOriginManifest { + origin_manifest_sequence, + origin_manifest_segment_cnt: self.segments_without_seq.len(), + version: self.version, + target_duration: self.target_duration, + discontinuity_sequence: self.discontinuity_sequence, + independent_segments: self.independent_segments, + maps: self.maps, + segments: self.segments_without_seq, + }) + } +} + +trait ParseNumericTag { + fn parse_numeric(&self, feature: &'static str) -> Result + where + T: std::str::FromStr; +} + +impl ParseNumericTag for str { + fn parse_numeric(&self, feature: &'static str) -> Result + where + T: std::str::FromStr, + { + self.trim().parse().map_err(|_| parser_feature(feature)) + } +} + +fn tag_name(line: &str) -> &str { line.split_once(':').map_or(line, |(tag, _)| tag) } + +fn is_allowed_normal_timeline_tag(tag: &str) -> bool { + matches!( + tag, + "#EXTM3U" + | "#EXT-X-VERSION" + | "#EXT-X-TARGETDURATION" + | "#EXT-X-MEDIA-SEQUENCE" + | "#EXTINF" + | "#EXT-X-DISCONTINUITY" + | "#EXT-X-DISCONTINUITY-SEQUENCE" + | "#EXT-X-MAP" + | "#EXT-X-BYTERANGE" + | "#EXT-X-PROGRAM-DATE-TIME" + | "#EXT-X-DATERANGE" + | "#EXT-X-INDEPENDENT-SEGMENTS" + | "#EXT-X-ALLOW-CACHE" + ) +} + +fn parse_tag_value<'a>(line: &'a str, tag: &str) -> Result<&'a str, OriginManifestTransientReason> { + line.strip_prefix(tag) + .and_then(|value| value.strip_prefix(':')) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| parser_feature("missing_tag_value")) +} + +fn parse_extinf_duration_ms(line: &str) -> Result { + let value = parse_tag_value(line, "#EXTINF")? + .split_once(',') + .map_or_else(|| parse_tag_value(line, "#EXTINF").unwrap_or_default(), |(duration, _)| duration) + .trim(); + parse_decimal_seconds_to_ms(value).ok_or_else(|| parser_feature("invalid_extinf")) +} + +fn parse_decimal_seconds_to_ms(value: &str) -> Option { + let (whole, fraction) = value.split_once('.').unwrap_or((value, "")); + if whole.is_empty() || !whole.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let whole_ms = whole.parse::().ok()?.checked_mul(1_000)?; + let mut fraction_ms = 0_u64; + let mut multiplier = 100_u64; + for byte in fraction.bytes().take(3) { + if !byte.is_ascii_digit() { + return None; + } + fraction_ms = fraction_ms.checked_add(u64::from(byte - b'0').checked_mul(multiplier)?)?; + multiplier /= 10; + } + if fraction.bytes().skip(3).any(|byte| !byte.is_ascii_digit()) { + return None; + } + whole_ms.checked_add(fraction_ms) +} + +fn parse_byterange_tag(line: &str) -> Result { + parse_byterange_value(parse_tag_value(line, "#EXT-X-BYTERANGE")?) +} + +fn parse_map_byterange(value: &str) -> Result { + match parse_byterange_value(value)? { + PendingByteRange::WithOffset(byte_range) => Ok(byte_range), + PendingByteRange::WithoutOffset(_) => Err(parser_feature("map_byterange_without_offset")), + } +} + +fn parse_byterange_value(value: &str) -> Result { + let trimmed = value.trim().trim_matches('"'); + if let Some((length, offset)) = trimmed.split_once('@') { + return Ok(PendingByteRange::WithOffset(ParsedByteRange { + length: length.parse_numeric("invalid_byterange_length")?, + offset: offset.parse_numeric("invalid_byterange_offset")?, + })); + } + Ok(PendingByteRange::WithoutOffset(trimmed.parse_numeric("invalid_byterange_length")?)) +} + +fn parse_attribute<'a>(line: &'a str, name: &str) -> Option<&'a str> { + line.split_once(':')? + .1 + .split(',') + .filter_map(|part| part.split_once('=')) + .find_map(|(key, value)| (key.trim() == name).then(|| value.trim())) +} + +fn resolve_uri(base: &str, reference: &str) -> String { + match rewrite_hls_url(base, reference) { + Cow::Borrowed(value) => value.to_string(), + Cow::Owned(value) => value, + } +} + +fn parser_feature(feature: &str) -> OriginManifestTransientReason { + OriginManifestTransientReason::ParserUnsupportedFeature { feature: feature.to_string() } +} + +#[cfg(test)] +mod tests { + use super::{ + OriginManifestParseOutcome, OriginManifestTransientReason, ParsedByteRange, ParsedManifestValiditySource, + parse_manifest_timing, parse_manifest_validity, parse_origin_manifest_timeline, parse_origin_media_manifest, + }; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn normal_manifest(body: &str) -> super::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } + } + + #[test] + fn missing_media_sequence_defaults_to_zero() { + let manifest = normal_manifest("#EXTM3U\n#EXTINF:4.0,\nseg001.ts\n"); + + assert_eq!(manifest.origin_manifest_sequence, 0); + assert_eq!(manifest.segments[0].origin_seq, 0); + } + + #[test] + fn present_media_sequence_is_used() { + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:322\n#EXTINF:4.0,\nseg001.ts\n"); + + assert_eq!(manifest.origin_manifest_sequence, 322); + assert_eq!(manifest.segments[0].origin_seq, 322); + } + + #[test] + fn timeline_markers_parse_media_sequence_through_transient_tags() { + let timeline = parse_origin_manifest_timeline( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:226\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg001.ts\n#EXTINF:4.0,\nseg002.ts\n", + ) + .expect("timeline markers parse"); + + assert_eq!(timeline.origin_manifest_sequence, 226); + assert_eq!(timeline.origin_manifest_segment_cnt, 2); + assert_eq!(timeline.origin_highwater(), Some(227)); + } + + #[test] + fn timeline_markers_default_missing_media_sequence_to_zero() { + let timeline = parse_origin_manifest_timeline( + "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ) + .expect("timeline markers parse"); + + assert_eq!(timeline.origin_manifest_sequence, 0); + assert_eq!(timeline.origin_manifest_segment_cnt, 1); + } + + #[test] + fn extinf_and_segment_uri_create_segment_model() { + let manifest = normal_manifest("#EXTM3U\n#EXTINF:4.567,\nmedia/seg001.ts\n"); + let segment = &manifest.segments[0]; + + assert_eq!(manifest.origin_manifest_segment_cnt, 1); + assert_eq!(segment.duration_ms, 4_567); + assert_eq!(segment.resolved_origin_url, "http://origin.example.com/live/final/media/seg001.ts"); + } + + #[test] + fn ext_x_key_triggers_transient_passthrough() { + let outcome = parse_origin_media_manifest("#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n", BASE_URL); + + assert_eq!( + outcome, + OriginManifestParseOutcome::TransientPassthrough { reason: OriginManifestTransientReason::ExtXKey } + ); + } + + #[test] + fn unsupported_tag_triggers_transient_passthrough() { + let outcome = parse_origin_media_manifest("#EXTM3U\n#EXT-X-PART:DURATION=1.0,URI=\"part.ts\"\n", BASE_URL); + + assert_eq!( + outcome, + OriginManifestParseOutcome::TransientPassthrough { + reason: OriginManifestTransientReason::UnsupportedTag { tag: "#EXT-X-PART".to_string() } + } + ); + } + + #[test] + fn ext_x_map_is_parsed_with_resolved_uri() { + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nseg001.m4s\n"); + + assert_eq!(manifest.maps[0].resolved_origin_uri, "http://origin.example.com/live/final/init.mp4"); + assert_eq!(manifest.segments[0].map_ref, Some(0)); + } + + #[test] + fn ext_x_map_relative_uri_is_resolved_against_final_manifest_url_after_redirect() { + let final_manifest_url = "https://cdn.example.net/live/redirected/playlist.m3u8"; + let outcome = + parse_origin_media_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nseg001.m4s\n", final_manifest_url); + let OriginManifestParseOutcome::Normal(manifest) = outcome else { + panic!("manifest should parse as normal timeline"); + }; + + assert_eq!(manifest.maps[0].resolved_origin_uri, "https://cdn.example.net/live/redirected/init.mp4"); + assert_eq!(manifest.segments[0].resolved_origin_url, "https://cdn.example.net/live/redirected/seg001.m4s"); + } + + #[test] + fn byterange_with_explicit_offset_is_parsed() { + let manifest = normal_manifest("#EXTM3U\n#EXT-X-BYTERANGE:500@1000\n#EXTINF:4.0,\nbig.m4s\n"); + + assert_eq!(manifest.segments[0].origin_byte_range, Some(ParsedByteRange { length: 500, offset: 1_000 })); + } + + #[test] + fn byterange_without_offset_uses_previous_offset_for_same_uri() { + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-BYTERANGE:500@1000\n#EXTINF:4.0,\nbig.m4s\n#EXT-X-BYTERANGE:250\n#EXTINF:4.0,\nbig.m4s\n", + ); + + assert_eq!(manifest.segments[1].origin_byte_range, Some(ParsedByteRange { length: 250, offset: 1_500 })); + } + + #[test] + fn byterange_without_previous_offset_fails_controlled() { + let outcome = parse_origin_media_manifest("#EXTM3U\n#EXT-X-BYTERANGE:250\n#EXTINF:4.0,\nbig.m4s\n", BASE_URL); + + assert_eq!( + outcome, + OriginManifestParseOutcome::TransientPassthrough { + reason: OriginManifestTransientReason::ParserUnsupportedFeature { + feature: "byterange_without_resolvable_offset".to_string() + } + } + ); + } + + #[test] + fn byterange_offset_overflow_triggers_parser_unsupported_feature() { + let body = "#EXTM3U\n#EXT-X-BYTERANGE:2@18446744073709551614\n#EXTINF:4.0,\nbig.m4s\n"; + let outcome = parse_origin_media_manifest(body, BASE_URL); + + assert_eq!( + outcome, + OriginManifestParseOutcome::TransientPassthrough { + reason: OriginManifestTransientReason::ParserUnsupportedFeature { + feature: "byterange_offset_overflow".to_string() + } + } + ); + } + + #[test] + fn discontinuity_and_discontinuity_sequence_are_parsed() { + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:7\n#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\nseg001.ts\n", + ); + + assert_eq!(manifest.discontinuity_sequence, Some(7)); + assert!(manifest.segments[0].discontinuity_before); + } + + #[test] + fn program_date_time_and_daterange_attach_to_next_segment() { + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-PROGRAM-DATE-TIME:2026-05-27T10:11:12Z\n#EXT-X-DATERANGE:ID=\"ad-1\",START-DATE=\"2026-05-27T10:11:12Z\"\n#EXTINF:4.0,\nseg001.ts\n", + ); + let segment = &manifest.segments[0]; + + assert_eq!(segment.program_date_time.as_deref(), Some("2026-05-27T10:11:12Z")); + assert_eq!( + segment.daterange_tags_before, + vec!["#EXT-X-DATERANGE:ID=\"ad-1\",START-DATE=\"2026-05-27T10:11:12Z\"".to_string()] + ); + } + + #[test] + fn version_and_target_duration_are_parsed() { + let manifest = + normal_manifest("#EXTM3U\n#EXT-X-VERSION:6\n#EXT-X-TARGETDURATION:10\n#EXTINF:4.0,\nseg001.ts\n"); + + assert_eq!(manifest.version, Some(6)); + assert_eq!(manifest.target_duration, Some(10)); + } + + #[test] + fn manifest_timing_parses_target_duration_and_last_extinf() { + let timing = parse_manifest_timing( + "#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:4.000,\na.ts\n#EXT-X-KEY:METHOD=AES-128,URI=\"k\"\n#EXTINF:6.250,\nb.ts\n", + ); + + assert_eq!(timing.target_duration_ms, Some(10_000)); + assert_eq!(timing.last_segment_duration_ms, Some(6_250)); + } + + #[test] + fn manifest_timing_normalizes_extinf_fraction_to_milliseconds() { + assert_eq!( + parse_manifest_timing("#EXTM3U\n#EXTINF:9.23,title\nseg.ts\n").last_segment_duration_ms, + Some(9_230) + ); + assert_eq!(parse_manifest_timing("#EXTM3U\n#EXTINF:7,\nseg.ts\n").last_segment_duration_ms, Some(7_000)); + } + + #[test] + fn manifest_validity_prefers_extinf_sum() { + let validity = parse_manifest_validity( + "#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:4.000,\na.ts\n#EXTINF:6.250,\nb.ts\n", + ) + .expect("validity"); + + assert_eq!(validity.playlist_duration_ms, 10_250); + assert_eq!(validity.segment_count, 2); + assert_eq!(validity.source, ParsedManifestValiditySource::ExtInfSum); + } + + #[test] + fn manifest_validity_falls_back_to_target_duration() { + let validity = parse_manifest_validity("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXTINF:x,\na.ts\n#EXTINF:y,\nb.ts\n") + .expect("validity"); + + assert_eq!(validity.playlist_duration_ms, 24_000); + assert_eq!(validity.segment_count, 2); + assert_eq!(validity.source, ParsedManifestValiditySource::TargetDuration); + } + + #[test] + fn manifest_validity_falls_back_to_segment_count_times_15_seconds() { + let validity = parse_manifest_validity("#EXTM3U\n#EXTINF:x,\na.ts\n#EXTINF:y,\nb.ts\n").expect("validity"); + + assert_eq!(validity.playlist_duration_ms, 30_000); + assert_eq!(validity.segment_count, 2); + assert_eq!(validity.source, ParsedManifestValiditySource::Fallback); + } +} diff --git a/backend/src/processing/parser/hls/transient_manifest.rs b/backend/src/processing/parser/hls/transient_manifest.rs new file mode 100644 index 000000000..f5516fd08 --- /dev/null +++ b/backend/src/processing/parser/hls/transient_manifest.rs @@ -0,0 +1,911 @@ +use super::rewrite_hls_url; +use crate::api::model::{ + ProxySessionId, TransientResourceId, TransientResourceKind, TransientResourceRef, + HLS_ACCESS_LEASE_ID_PLACEHOLDER, +}; +use crate::model::StripConfig; +use shared::model::HlsStripMode; +use shared::utils::CONSTANTS; +use std::{collections::HashMap, time::Duration}; +use url::Url; + +pub(super) const MIN_HLS_INITIAL_VISIBLE_SEGMENTS: usize = 3; + +/// Result of a transient passthrough manifest rewrite. +#[derive(Debug, Clone)] +pub struct TransientRewriteResult { + pub body: String, + pub resources: Vec, +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct TransientRewriteOptions { + pub handoff_discontinuity_sequence: Option, +} + +/// Rewrites only HLS URI surfaces to transient live-HLS proxy resources. +pub struct TransientManifestRewriter; + +impl TransientManifestRewriter { + /// Rewrites HLS URI surfaces to transient proxy resources. + /// + /// `final_manifest_url` must be the concrete URL of the manifest that was actually fetched. For provider-url + /// failover and HTTP redirects this can be the selected mirror or final CDN/origin host. Relative segment, MAP and + /// key URIs are resolved against this URL so transient resource fetches keep working after redirects. + pub fn rewrite( + body: &str, + final_manifest_url: &str, + proxy_session_id: &ProxySessionId, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + ) -> TransientRewriteResult { + Self::rewrite_with_options( + body, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + TransientRewriteOptions::default(), + ) + } + + /// Rewrites HLS URI surfaces to transient proxy resources with extra rendering options. + /// + /// Keep `final_manifest_url` aligned with the actual fetched manifest URL. Passing the original `provider://` input + /// URL or a pre-redirect URL would make relative transient resources point at the wrong fetch target. + pub fn rewrite_with_options( + body: &str, + final_manifest_url: &str, + proxy_session_id: &ProxySessionId, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + options: TransientRewriteOptions, + ) -> TransientRewriteResult { + let mut rewritten_body = String::with_capacity(body.len()); + let mut resources = HashMap::::new(); + + for part in body.split_inclusive('\n') { + let (line, line_ending) = split_line_ending(part); + let rewritten_line = rewrite_line( + line, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + &mut resources, + ); + rewritten_body.push_str(&rewritten_line); + rewritten_body.push_str(line_ending); + } + + if body.is_empty() { + return TransientRewriteResult { + body: rewritten_body, + resources: Vec::new(), + }; + } + + if let Some(discontinuity_sequence) = options.handoff_discontinuity_sequence { + rewritten_body = apply_handoff_discontinuity_boundary(&rewritten_body, discontinuity_sequence); + } + + TransientRewriteResult { + body: rewritten_body, + resources: resources.into_values().collect(), + } + } +} + +pub fn materialize_transient_provisioning_handoff_view( + origin_body: &str, + previous_provisioning_body: Option<&str>, + strip: &StripConfig, + provisioning_segment_duration_ms: u64, +) -> Option { + let previous_provisioning_body = previous_provisioning_body?; + let origin_lines = manifest_lines(origin_body); + let origin_units = media_segment_units(&origin_lines); + if origin_units.is_empty() { + return None; + } + + let origin_window_segments = configured_strip_segments(strip, &origin_lines, &origin_units) + .saturating_add(MIN_HLS_INITIAL_VISIBLE_SEGMENTS) + .min(origin_units.len()); + let origin_window_start = origin_units.len().saturating_sub(origin_window_segments); + let origin_batch_segments = origin_window_segments.min(MIN_HLS_INITIAL_VISIBLE_SEGMENTS); + if origin_batch_segments == 0 { + return None; + } + let selected_origin_units = &origin_units[origin_window_start..origin_window_start + origin_batch_segments]; + + let previous_lines = manifest_lines(previous_provisioning_body); + let previous_units = media_segment_units(&previous_lines); + if previous_units.is_empty() { + return None; + } + let provisioning_tail_segments = + TARGET_TRANSIENT_HANDOFF_SEGMENTS.saturating_sub(origin_batch_segments).saturating_sub(1); + let selected_previous_units = previous_units + .len() + .checked_sub(provisioning_tail_segments) + .map_or(previous_units.as_slice(), |start| &previous_units[start..]); + let gap_unit = selected_previous_units.last().copied().or_else(|| previous_units.last().copied())?; + + let first_origin_unit = origin_units[0]; + let media_sequence = handoff_provisioning_media_sequence( + &previous_lines, + &previous_units, + previous_units.len().saturating_sub(selected_previous_units.len()), + ); + let mut rewritten = String::with_capacity( + origin_body + .len() + .saturating_add(previous_provisioning_body.len()) + .saturating_add(64), + ); + for line in &origin_lines[..first_origin_unit.start] { + if line.line.trim().starts_with("#EXT-X-MEDIA-SEQUENCE:") { + if let Some(media_sequence) = media_sequence { + let _ = std::fmt::Write::write_fmt( + &mut rewritten, + format_args!("#EXT-X-MEDIA-SEQUENCE:{media_sequence}{}", line.ending), + ); + continue; + } + } + rewritten.push_str(line.line); + rewritten.push_str(line.ending); + } + append_manifest_block_separator(&mut rewritten); + for unit in selected_previous_units { + append_media_unit_with_duration_override( + &mut rewritten, + &previous_lines, + *unit, + Some(provisioning_segment_duration_ms), + ); + } + append_manifest_block_separator(&mut rewritten); + append_gap_media_unit(&mut rewritten, &previous_lines, gap_unit, provisioning_segment_duration_ms); + append_manifest_block_separator(&mut rewritten); + for (index, unit) in selected_origin_units.iter().enumerate() { + if index == 0 && !media_unit_has_discontinuity(&origin_lines, *unit) { + rewritten.push_str("#EXT-X-DISCONTINUITY\n"); + } + append_media_unit_with_duration_override(&mut rewritten, &origin_lines, *unit, None); + } + + Some(rewritten) +} + +pub fn apply_transient_discontinuity_sequence(body: &str, discontinuity_sequence: u64) -> String { + let lines = manifest_lines(body); + let existing_sequence_index = lines.iter().position(|line| is_discontinuity_sequence_tag(line.line.trim())); + let insert_sequence_index = existing_sequence_index.unwrap_or_else(|| discontinuity_sequence_insert_index(&lines)); + + let mut rewritten = String::with_capacity(body.len().saturating_add(40)); + for index in 0..=lines.len() { + if existing_sequence_index.is_none() && index == insert_sequence_index { + let _ = std::fmt::Write::write_fmt( + &mut rewritten, + format_args!("#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}\n"), + ); + } + if index == lines.len() { + break; + } + if existing_sequence_index == Some(index) { + let _ = std::fmt::Write::write_fmt( + &mut rewritten, + format_args!("#EXT-X-DISCONTINUITY-SEQUENCE:{}{}", discontinuity_sequence, lines[index].ending), + ); + } else { + rewritten.push_str(lines[index].line); + rewritten.push_str(lines[index].ending); + } + } + rewritten +} + +pub fn transient_discontinuity_sequence(body: &str) -> Option { + manifest_lines(body) + .iter() + .find_map(|line| parse_discontinuity_sequence(line.line)) +} + +pub fn transient_visible_discontinuity_count(body: &str) -> u64 { + u64::try_from( + manifest_lines(body) + .iter() + .filter(|line| is_discontinuity_tag(line.line.trim())) + .count(), + ) + .unwrap_or(u64::MAX) +} + +fn rewrite_line( + line: &str, + final_manifest_url: &str, + proxy_session_id: &ProxySessionId, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + resources: &mut HashMap, +) -> String { + let trimmed = line.trim(); + if trimmed.is_empty() { + return line.to_string(); + } + + if trimmed.starts_with("#EXT-X-KEY:") { + return rewrite_uri_attribute( + line, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + TransientResourceKind::Key, + ); + } + + if trimmed.starts_with("#EXT-X-MAP:") { + return rewrite_uri_attribute( + line, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + TransientResourceKind::Map, + ); + } + + if trimmed.starts_with('#') && CONSTANTS.re_hls_uri.is_match(line) { + return rewrite_uri_attribute( + line, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + TransientResourceKind::Other, + ); + } + + if trimmed.starts_with('#') { + return line.to_string(); + } + + rewrite_resource_uri( + trimmed, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + TransientResourceKind::Segment, + ) + .0 +} + +#[allow(clippy::too_many_arguments)] +fn rewrite_uri_attribute( + line: &str, + final_manifest_url: &str, + proxy_session_id: &ProxySessionId, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + resources: &mut HashMap, + kind: TransientResourceKind, +) -> String { + let Some(caps) = CONSTANTS.re_hls_uri.captures(line) else { + return line.to_string(); + }; + let uri = &caps[1]; + let (proxy_uri, _) = rewrite_resource_uri( + uri, + final_manifest_url, + proxy_session_id, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + resources, + kind, + ); + CONSTANTS + .re_hls_uri + .replace(line, format!(r#"URI="{proxy_uri}""#)) + .to_string() +} + +#[allow(clippy::too_many_arguments)] +fn rewrite_resource_uri( + uri: &str, + final_manifest_url: &str, + proxy_session_id: &ProxySessionId, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + resources: &mut HashMap, + kind: TransientResourceKind, +) -> (String, TransientResourceId) { + // Resolve against the final fetched manifest URL, not the configured provider:// entry or pre-redirect URL. Relative + // resources commonly live under the redirect/CDN host returned by the manifest request. + let resolved_origin_uri = rewrite_hls_url(final_manifest_url, uri).into_owned(); + let extension = extension_for_resource(&resolved_origin_uri, kind); + let resource = TransientResourceRef::new( + kind, + resolved_origin_uri, + reverse_proxy_rewrite_secret, + now_ms, + ttl_ms, + Some(extension.clone()), + ); + let resource_id = resource.id.clone(); + resources + .entry(resource_id.clone()) + .and_modify(|existing| *existing = resource.clone()) + .or_insert(resource); + ( + format!( + "/hls/shared/live/{}/{}/r/{}.{}", + proxy_session_id.0, HLS_ACCESS_LEASE_ID_PLACEHOLDER, resource_id.0, extension + ), + resource_id, + ) +} + +fn extension_for_resource(resolved_origin_uri: &str, kind: TransientResourceKind) -> String { + extract_extension(resolved_origin_uri) + .filter(|extension| extension.bytes().all(|byte| byte.is_ascii_alphanumeric())) + .map_or_else(|| fallback_extension(kind).to_string(), |extension| extension.to_ascii_lowercase()) +} + +fn extract_extension(resolved_origin_uri: &str) -> Option { + if let Ok(url) = Url::parse(resolved_origin_uri) { + return url + .path_segments() + .and_then(Iterator::last) + .and_then(|file_name| file_name.rsplit_once('.').map(|(_, extension)| extension.to_string())) + .filter(|extension| !extension.is_empty()); + } + + let without_query = resolved_origin_uri + .split_once('?') + .map_or(resolved_origin_uri, |(path, _)| path); + let without_fragment = without_query + .split_once('#') + .map_or(without_query, |(path, _)| path); + without_fragment + .rsplit_once('.') + .map(|(_, extension)| extension.to_string()) + .filter(|extension| !extension.is_empty()) +} + +fn fallback_extension(kind: TransientResourceKind) -> &'static str { + match kind { + TransientResourceKind::Key => "key", + TransientResourceKind::Map => "mp4", + TransientResourceKind::Segment | TransientResourceKind::Other => "bin", + } +} + +fn split_line_ending(part: &str) -> (&str, &str) { + if let Some(line) = part.strip_suffix("\r\n") { + (line, "\r\n") + } else if let Some(line) = part.strip_suffix('\n') { + (line, "\n") + } else { + (part, "") + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct ManifestLine<'a> { + pub(super) line: &'a str, + pub(super) ending: &'a str, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct MediaSegmentUnit { + pub(super) start: usize, + pub(super) end: usize, +} + +impl MediaSegmentUnit { + pub(super) const fn contains(self, index: usize) -> bool { self.start <= index && index <= self.end } +} + +pub(super) fn manifest_lines(body: &str) -> Vec> { + body.split_inclusive('\n') + .map(|part| { + let (line, ending) = split_line_ending(part); + ManifestLine { line, ending } + }) + .collect() +} + +pub(super) fn media_segment_units(lines: &[ManifestLine<'_>]) -> Vec { + let mut units = Vec::new(); + let mut unit_start = None; + for (index, line) in lines.iter().enumerate() { + let trimmed = line.line.trim(); + if is_segment_unit_tag(trimmed) { + unit_start.get_or_insert(index); + continue; + } + if trimmed.is_empty() { + continue; + } + if trimmed.starts_with('#') { + unit_start = None; + continue; + } + let start = unit_start.take().unwrap_or(index); + units.push(MediaSegmentUnit { start, end: index }); + } + units +} + +pub(super) fn configured_strip_segments( + strip: &StripConfig, + lines: &[ManifestLine<'_>], + units: &[MediaSegmentUnit], +) -> usize { + match strip.mode { + HlsStripMode::Segments => usize::try_from(strip.value).unwrap_or(usize::MAX), + HlsStripMode::Seconds => { + let target_ms = strip.value.saturating_mul(1_000); + let mut accumulated_ms = 0_u64; + let mut strip_segments = 0_usize; + for unit in units.iter().rev() { + strip_segments = strip_segments.saturating_add(1); + accumulated_ms = accumulated_ms.saturating_add(segment_duration_ms(lines, *unit)); + if accumulated_ms >= target_ms { + break; + } + } + strip_segments + } + } +} + +fn segment_duration_ms(lines: &[ManifestLine<'_>], unit: MediaSegmentUnit) -> u64 { + lines[unit.start..=unit.end] + .iter() + .find_map(|line| { + let trimmed = line.line.trim(); + let extinf_value = trimmed.strip_prefix("#EXTINF:")?; + let duration = extinf_value.split_once(',').map_or(extinf_value, |(value, _)| value); + duration_ms_from_extinf(duration) + }) + .unwrap_or(0) +} + +const TARGET_TRANSIENT_HANDOFF_SEGMENTS: usize = 6; + +fn append_media_unit_with_duration_override( + output: &mut String, + lines: &[ManifestLine<'_>], + unit: MediaSegmentUnit, + duration_ms: Option, +) { + for line in &lines[unit.start..=unit.end] { + if line.line.trim().starts_with("#EXTINF:") { + if let Some(duration_ms) = duration_ms { + let _ = std::fmt::Write::write_fmt( + output, + format_args!("#EXTINF:{},{}", format_duration_ms(duration_ms), line.ending), + ); + continue; + } + } + output.push_str(line.line); + output.push_str(line.ending); + } +} + +fn append_gap_media_unit( + output: &mut String, + lines: &[ManifestLine<'_>], + unit: MediaSegmentUnit, + duration_ms: u64, +) { + output.push_str("#EXT-X-GAP\n"); + let uri_index = lines[unit.start..=unit.end] + .iter() + .rposition(|line| { + let trimmed = line.line.trim(); + !trimmed.is_empty() && !trimmed.starts_with('#') + }) + .map(|relative_index| unit.start + relative_index); + if let Some(uri_index) = uri_index { + let _ = std::fmt::Write::write_fmt( + output, + format_args!("#EXTINF:{},{}", format_duration_ms(duration_ms), lines[uri_index].ending), + ); + output.push_str(lines[uri_index].line); + output.push_str(lines[uri_index].ending); + } +} + +fn append_manifest_block_separator(output: &mut String) { + if output.is_empty() || output.ends_with("\n\n") { + return; + } + if !output.ends_with('\n') { + output.push('\n'); + } + output.push('\n'); +} + +fn media_unit_has_discontinuity(lines: &[ManifestLine<'_>], unit: MediaSegmentUnit) -> bool { + lines[unit.start..=unit.end] + .iter() + .any(|line| is_discontinuity_tag(line.line.trim())) +} + +fn handoff_provisioning_media_sequence( + lines: &[ManifestLine<'_>], + units: &[MediaSegmentUnit], + selected_unit_start: usize, +) -> Option { + let media_sequence = lines.iter().find_map(|line| parse_media_sequence(line.line))?; + if selected_unit_start >= units.len() { + return None; + } + Some(media_sequence.saturating_add(u64::try_from(selected_unit_start).ok()?)) +} + +fn parse_media_sequence(line: &str) -> Option { + line.trim().strip_prefix("#EXT-X-MEDIA-SEQUENCE:")?.trim().parse().ok() +} + +fn format_duration_ms(duration_ms: u64) -> String { format!("{}.{:03}", duration_ms / 1_000, duration_ms % 1_000) } + +fn apply_handoff_discontinuity_boundary(body: &str, handoff_discontinuity_sequence: u64) -> String { + let lines = manifest_lines(body); + let origin_discontinuity_sequence = lines + .iter() + .find_map(|line| parse_discontinuity_sequence(line.line)) + .unwrap_or(0); + let effective_discontinuity_sequence = + origin_discontinuity_sequence.saturating_add(handoff_discontinuity_sequence); + let existing_sequence_index = lines.iter().position(|line| is_discontinuity_sequence_tag(line.line.trim())); + let insert_sequence_index = existing_sequence_index.unwrap_or_else(|| discontinuity_sequence_insert_index(&lines)); + let first_segment = first_media_segment_boundary(&lines); + + let mut rewritten = String::with_capacity(body.len().saturating_add(64)); + for index in 0..=lines.len() { + if existing_sequence_index.is_none() && index == insert_sequence_index { + let _ = std::fmt::Write::write_fmt( + &mut rewritten, + format_args!("#EXT-X-DISCONTINUITY-SEQUENCE:{effective_discontinuity_sequence}\n"), + ); + } + if let Some((boundary_index, has_discontinuity)) = first_segment { + if !has_discontinuity && index == boundary_index { + rewritten.push_str("#EXT-X-DISCONTINUITY\n"); + } + } + if index == lines.len() { + break; + } + if existing_sequence_index == Some(index) { + let _ = std::fmt::Write::write_fmt( + &mut rewritten, + format_args!( + "#EXT-X-DISCONTINUITY-SEQUENCE:{}{}", + effective_discontinuity_sequence, lines[index].ending + ), + ); + } else { + rewritten.push_str(lines[index].line); + rewritten.push_str(lines[index].ending); + } + } + rewritten +} + +fn discontinuity_sequence_insert_index(lines: &[ManifestLine<'_>]) -> usize { + lines + .iter() + .position(|line| line.line.trim().starts_with("#EXT-X-MEDIA-SEQUENCE:")) + .or_else(|| lines.iter().position(|line| line.line.trim().starts_with("#EXT-X-TARGETDURATION:"))) + .or_else(|| lines.iter().position(|line| line.line.trim() == "#EXTM3U")) + .map_or(0, |index| index + 1) +} + +fn first_media_segment_boundary(lines: &[ManifestLine<'_>]) -> Option<(usize, bool)> { + let mut unit_start = None; + let mut unit_has_discontinuity = false; + for (index, line) in lines.iter().enumerate() { + let trimmed = line.line.trim(); + if trimmed.is_empty() { + continue; + } + if is_media_segment_unit_tag(trimmed) { + unit_start.get_or_insert(index); + if is_discontinuity_tag(trimmed) { + unit_has_discontinuity = true; + } + continue; + } + if trimmed.starts_with('#') { + unit_start = None; + unit_has_discontinuity = false; + continue; + } + return Some((unit_start.unwrap_or(index), unit_has_discontinuity)); + } + None +} + +fn parse_discontinuity_sequence(line: &str) -> Option { + line.trim().strip_prefix("#EXT-X-DISCONTINUITY-SEQUENCE:")?.trim().parse().ok() +} + +fn is_discontinuity_sequence_tag(line: &str) -> bool { + line.starts_with("#EXT-X-DISCONTINUITY-SEQUENCE:") +} + +fn is_discontinuity_tag(line: &str) -> bool { line == "#EXT-X-DISCONTINUITY" } + +fn duration_ms_from_extinf(value: &str) -> Option { + let seconds = value.trim().parse::().ok()?; + if !seconds.is_finite() || seconds.is_sign_negative() { + return None; + } + u64::try_from(Duration::from_secs_f64(seconds).as_millis()).ok() +} + +fn is_segment_unit_tag(line: &str) -> bool { + is_media_segment_unit_tag(line) +} + +fn is_media_segment_unit_tag(line: &str) -> bool { + line.starts_with("#EXTINF:") + || line.starts_with("#EXT-X-BYTERANGE:") + || line.starts_with("#EXT-X-PROGRAM-DATE-TIME:") + || is_discontinuity_tag(line) +} + +pub(super) const fn strip_mode_log_value(mode: HlsStripMode) -> &'static str { + match mode { + HlsStripMode::Segments => "segments", + HlsStripMode::Seconds => "seconds", + } +} + +#[cfg(test)] +mod tests { + use super::{ + apply_transient_discontinuity_sequence, materialize_transient_provisioning_handoff_view, + transient_discontinuity_sequence, TransientManifestRewriter, TransientRewriteOptions, + }; + use crate::api::model::{build_transient_resource_id, ProxySessionId, TransientResourceKind}; + use crate::model::StripConfig; +use shared::model::HlsStripMode; + + const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + + fn rewrite(body: &str) -> crate::processing::parser::hls::transient_manifest::TransientRewriteResult { + TransientManifestRewriter::rewrite(body, BASE_URL, &ProxySessionId("proxy-id".to_string()), b"secret", 100, 1_000) + } + + fn rewrite_with_handoff(body: &str) -> crate::processing::parser::hls::transient_manifest::TransientRewriteResult { + TransientManifestRewriter::rewrite_with_options( + body, + BASE_URL, + &ProxySessionId("proxy-id".to_string()), + b"secret", + 100, + 1_000, + TransientRewriteOptions { + handoff_discontinuity_sequence: Some(0), + }, + ) + } + + #[test] + fn ext_x_key_uri_is_rewritten_without_changing_other_attributes() { + let result = rewrite("#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"keys/key.bin\",IV=0x1\n#EXTINF:4.0,\nseg.ts\n"); + + assert!(result.body.contains("#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result.body.contains(".bin\",IV=0x1")); + assert!(!result.body.contains("keys/key.bin")); + } + + #[test] + fn segment_uri_lines_are_rewritten_to_transient_resources() { + let result = rewrite("#EXTM3U\n#EXTINF:4.0,\nmedia/seg001.ts\n"); + + assert!(result.body.contains("/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result.body.contains(".ts")); + assert!(!result.body.contains("media/seg001.ts")); + } + + #[test] + fn ext_x_map_uri_is_rewritten_without_changing_other_attributes() { + let result = rewrite("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\",BYTERANGE=\"10@5\"\n#EXTINF:4.0,\nseg.m4s\n"); + + assert!(result.body.contains("#EXT-X-MAP:URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result.body.contains(".mp4\",BYTERANGE=\"10@5\"")); + assert!(!result.body.contains("init.mp4")); + } + + #[test] + fn byterange_and_media_sequence_remain_unchanged() { + let result = rewrite("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:42\n#EXT-X-BYTERANGE:100@200\n#EXTINF:4.0,\nseg.ts\n"); + + assert!(result.body.contains("#EXT-X-MEDIA-SEQUENCE:42")); + assert!(result.body.contains("#EXT-X-BYTERANGE:100@200")); + } + + #[test] + fn transient_output_never_uses_legacy_hls_token_route() { + let result = rewrite( + "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + assert!(!result.body.contains("/hls/user/pass/")); + assert!(!result.body.contains("key.bin")); + assert!(!result.body.contains("init.mp4")); + assert!(!result.body.contains("seg.ts")); + assert_eq!(result.body.matches("/hls/shared/live/proxy-id/__hls_access_lease_id__/r/").count(), 3); + } + + #[test] + fn unsupported_uri_attributes_are_rewritten_to_transient_resources() { + let result = rewrite("#EXTM3U\n#EXT-X-PART:DURATION=1.0,URI=\"parts/part001.m4s\"\n"); + + assert!(result.body.contains("#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/__hls_access_lease_id__/r/")); + assert!(result.body.contains(".m4s\"")); + assert!(!result.body.contains("parts/part001.m4s")); + } + + #[test] + fn same_origin_uri_uses_same_transient_resource_id() { + let result = rewrite("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n#EXTINF:4.0,\n./seg.ts\n"); + let expected = build_transient_resource_id("http://origin.example.com/live/final/seg.ts", b"secret"); + + assert_eq!(result.resources.len(), 1); + assert_eq!(result.resources[0].id, expected); + assert_eq!(result.body.matches(&expected.0).count(), 2); + } + + #[test] + fn relative_resources_use_final_manifest_url_after_http_redirect() { + let final_manifest_url = "https://cdn-final.example.net/redirected/live/index.m3u8"; + let result = TransientManifestRewriter::rewrite( + "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"keys/key.bin\"\n#EXT-X-MAP:URI=\"init/init.mp4\"\n#EXTINF:4.0,\nmedia/seg001.ts\n", + final_manifest_url, + &ProxySessionId("proxy-id".to_string()), + b"secret", + 100, + 1_000, + ); + + let key_uri = "https://cdn-final.example.net/redirected/live/keys/key.bin"; + let map_uri = "https://cdn-final.example.net/redirected/live/init/init.mp4"; + let segment_uri = "https://cdn-final.example.net/redirected/live/media/seg001.ts"; + let expected_segment_id = build_transient_resource_id(segment_uri, b"secret"); + + assert_eq!(result.resources.len(), 3); + assert!(result.resources.iter().any(|resource| { + resource.kind == TransientResourceKind::Key && resource.resolved_origin_uri == key_uri + })); + assert!(result.resources.iter().any(|resource| { + resource.kind == TransientResourceKind::Map && resource.resolved_origin_uri == map_uri + })); + let segment = result + .resources + .iter() + .find(|resource| resource.kind == TransientResourceKind::Segment) + .expect("segment resource"); + assert_eq!(segment.resolved_origin_uri, segment_uri); + assert_eq!(segment.id, expected_segment_id); + assert!(result.body.contains(&format!("/r/{}.ts", expected_segment_id.0))); + } + + #[test] + fn same_relative_resource_on_different_final_hosts_uses_distinct_transient_ids() { + let first = TransientManifestRewriter::rewrite( + "#EXTM3U\n#EXTINF:4.0,\nseg.ts\n", + "https://cdn-a.example.net/live/index.m3u8", + &ProxySessionId("proxy-id".to_string()), + b"secret", + 100, + 1_000, + ); + let second = TransientManifestRewriter::rewrite( + "#EXTM3U\n#EXTINF:4.0,\nseg.ts\n", + "https://cdn-b.example.net/live/index.m3u8", + &ProxySessionId("proxy-id".to_string()), + b"secret", + 100, + 1_000, + ); + + let first_resource = first.resources.first().expect("first resource"); + let second_resource = second.resources.first().expect("second resource"); + + assert_eq!(first_resource.resolved_origin_uri, "https://cdn-a.example.net/live/seg.ts"); + assert_eq!(second_resource.resolved_origin_uri, "https://cdn-b.example.net/live/seg.ts"); + assert_ne!(first_resource.id, second_resource.id); + } + + #[test] + fn handoff_boundary_sets_discontinuity_sequence_and_first_segment_discontinuity() { + let result = rewrite_with_handoff( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:7\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg.ts\n", + ); + + assert!(result.body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:7\n")); + assert!(result.body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\n")); + assert_eq!(result.body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + } + + #[test] + fn handoff_boundary_does_not_duplicate_existing_first_segment_discontinuity() { + let result = rewrite_with_handoff( + "#EXTM3U\n#EXT-X-DISCONTINUITY-SEQUENCE:7\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-DISCONTINUITY\n#EXTINF:4.0,\nseg.ts\n", + ); + + assert!(result.body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:7\n")); + assert_eq!(result.body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + } + + #[test] + fn transient_provisioning_handoff_view_keeps_tail_gap_and_origin_head() { + let previous = "#EXTM3U\n#EXT-X-VERSION:7\n#EXT-X-INDEPENDENT-SEGMENTS\n#EXT-X-TARGETDURATION:3\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-DISCONTINUITY-SEQUENCE:0\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000000.ts?pseq=0\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000001.ts?pseq=1\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000002.ts?pseq=2\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000003.ts?pseq=3\n"; + let rewritten_origin = rewrite_with_handoff( + "#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:2455\n#EXTINF:0.64,\na.ts\n#EXTINF:1.92,\nb.ts\n#EXTINF:0.84,\nc.ts\n#EXTINF:1.08,\nd.ts\n#EXTINF:0.56,\ne.ts\n#EXTINF:1.36,\nf.ts\n", + ); + + let body = materialize_transient_provisioning_handoff_view( + &rewritten_origin.body, + Some(previous), + &strip_segments(2), + 2_000, + ) + .expect("handoff view should render"); + + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:2\n")); + assert!(body.contains("#EXT-X-TARGETDURATION:2\n")); + assert!(!body.contains("#EXT-X-TARGETDURATION:3\n")); + assert!(body.contains("#EXT-X-DISCONTINUITY-SEQUENCE:0\n\n#EXTINF:2.000,")); + assert_eq!(body.matches("#EXTINF:").count(), 6); + assert!(body.contains("#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000002.ts?pseq=2")); + assert!(body.contains("\n#EXT-X-GAP\n#EXTINF:2.000,\n/hls/shared/live/proxy-id/__hls_access_lease_id__/000003.ts?pseq=3")); + assert!(body.contains("\n#EXT-X-DISCONTINUITY\n#EXTINF:1.92,")); + let gap = body.find("\n#EXT-X-GAP\n").expect("gap tag"); + let discontinuity = body.find("\n#EXT-X-DISCONTINUITY\n#EXTINF:1.92,").expect("handoff discontinuity"); + assert!(gap < discontinuity); + } + + #[test] + fn transient_discontinuity_sequence_is_kept_after_handoff() { + let body = apply_transient_discontinuity_sequence( + "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-MEDIA-SEQUENCE:6560\n#EXT-X-TARGETDURATION:12\n#EXTINF:9.6,\na.ts\n", + 1, + ); + + assert_eq!(transient_discontinuity_sequence(&body), Some(1)); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:6560\n#EXT-X-DISCONTINUITY-SEQUENCE:1\n")); + } + + fn strip_segments(value: u64) -> StripConfig { StripConfig { mode: HlsStripMode::Segments, value } } + +} diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 5f9a40054..5bfc24aa3 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -4,7 +4,8 @@ use shared::model::{ CatchupAttribute, CatchupProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, XtreamCluster, }; -use shared::utils::{default_supported_video_extensions, extract_id_from_url, extract_numeric_id_from_url, Internable}; +use shared::utils::{extract_id_from_url, extract_numeric_id_from_url, Internable}; +use shared::defaults::{default_supported_video_extensions}; use std::borrow::BorrowMut; use std::sync::Arc; use tokio::io::AsyncBufReadExt; diff --git a/backend/src/processing/processor/mod.rs b/backend/src/processing/processor/mod.rs index 7034c75d0..29d69c5b8 100644 --- a/backend/src/processing/processor/mod.rs +++ b/backend/src/processing/processor/mod.rs @@ -67,8 +67,8 @@ macro_rules! create_resolve_options_function_for_xtream_target { false, false, false, - shared::utils::default_resolve_delay_secs(), - shared::utils::default_resolve_background(), + shared::defaults::default_resolve_delay_secs(), + shared::defaults::default_resolve_background(), ) }; diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index a525361cc..74408aa94 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -48,9 +48,12 @@ use shared::{ SourceStats, StreamProperties, TargetStats, UUIDType, XtreamCluster, }, utils::{ - create_alias_uuid, default_as_default, default_probe_delay_secs, default_probe_live_interval, interner_gc, + create_alias_uuid, interner_gc, Internable, }, + defaults::{ + default_as_default, default_probe_delay_secs, default_probe_live_interval, + } }; use std::{ collections::{HashMap, HashSet}, diff --git a/backend/src/processing/processor/probe_handle_guard.rs b/backend/src/processing/processor/probe_handle_guard.rs index a0b3b7cca..8cb09d1a6 100644 --- a/backend/src/processing/processor/probe_handle_guard.rs +++ b/backend/src/processing/processor/probe_handle_guard.rs @@ -49,7 +49,8 @@ mod tests { use arc_swap::{ArcSwap, ArcSwapOption}; use shared::{ model::{ConfigPaths, InputFetchMethod, InputType}, - utils::{default_probe_user_priority, Internable}, + utils::{Internable}, + defaults::{default_probe_user_priority}, }; use std::{collections::HashMap, sync::Arc}; diff --git a/backend/src/processing/processor/resolve_options.rs b/backend/src/processing/processor/resolve_options.rs index c1968d352..a6883756e 100644 --- a/backend/src/processing/processor/resolve_options.rs +++ b/backend/src/processing/processor/resolve_options.rs @@ -23,7 +23,7 @@ impl Default for ResolveOptions { fn default() -> Self { Self { flags: ResolveOptionsFlags::Background.into(), - resolve_delay: shared::utils::default_resolve_delay_secs(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), } } } diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index 645189b34..780eecf9d 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -36,7 +36,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use shared::foundation::ValueProvider; -use shared::utils::default_probe_user_priority; +use shared::defaults::default_probe_user_priority; create_resolve_options_function_for_xtream_target!(series); diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 05e215606..b1231b79f 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -36,7 +36,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use shared::foundation::ValueProvider; -use shared::utils::default_probe_user_priority; +use shared::defaults::default_probe_user_priority; create_resolve_options_function_for_xtream_target!(vod); diff --git a/backend/src/repository/alias_repository.rs b/backend/src/repository/alias_repository.rs index 1cc62cbf0..7cf526f75 100644 --- a/backend/src/repository/alias_repository.rs +++ b/backend/src/repository/alias_repository.rs @@ -36,6 +36,29 @@ const DEFAULT_COLUMNS: &[&str] = &[ ]; const CSV_EXTENSION: &str = ".csv"; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AliasExpDateSortOrder { + NewestFirst, + OldestFirst, +} + +pub fn compare_alias_exp_date_with_order( + a: &ConfigInputAliasDto, + b: &ConfigInputAliasDto, + order: AliasExpDateSortOrder, +) -> std::cmp::Ordering { + match (a.exp_date, b.exp_date) { + (Some(a_ts), Some(b_ts)) => match order { + AliasExpDateSortOrder::NewestFirst => b_ts.cmp(&a_ts), + AliasExpDateSortOrder::OldestFirst => a_ts.cmp(&b_ts), + }, + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + .then_with(|| a.name.cmp(&b.name)) +} + pub fn is_csv_file(url: &str) -> bool { url.to_lowercase().ends_with(CSV_EXTENSION) } @@ -547,6 +570,7 @@ pub async fn csv_patch_batch_remove_expired( pub async fn csv_patch_batch_sort_by_exp_date( input_type: InputType, csv_path: &Path, + order: AliasExpDateSortOrder, ) -> Result { let (file_path, mut aliases) = csv_read_inputs_from_path(input_type, csv_path) .map_err(|err| TuliproxError::ConfigInput(format!("{err}"))) @@ -555,11 +579,7 @@ pub async fn csv_patch_batch_sort_by_exp_date( return Ok(false); } let mut sorted = aliases.clone(); - sorted.sort_by(|a, b| { - let a_ts = a.exp_date.unwrap_or(i64::MIN); - let b_ts = b.exp_date.unwrap_or(i64::MIN); - b_ts.cmp(&a_ts).then_with(|| a.name.cmp(&b.name)) - }); + sorted.sort_by(|a, b| compare_alias_exp_date_with_order(a, b, order)); if sorted == aliases { return Ok(false); } @@ -572,10 +592,12 @@ pub async fn csv_patch_batch_sort_by_exp_date( #[cfg(test)] mod tests { + use super::{csv_patch_batch_sort_by_exp_date, csv_read_inputs_from_path, AliasExpDateSortOrder}; use crate::repository::csv_read_inputs_from_reader; use crate::utils::{file_reader}; use shared::model::InputType; use std::io::Cursor; + use std::path::PathBuf; const M3U_BATCH: &str = r" #url;name;max_connections;priority @@ -638,4 +660,40 @@ input_2;de566567;de2345f43g5;http://provider_2.tv:8080;1;2028-12-23 13:12:34 assert!(!config.url.contains("username")); } } + + fn temp_csv_path(test_name: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "tuliprox-{test_name}-{}-{}.csv", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_nanos() + )) + } + + #[tokio::test] + async fn csv_sort_by_exp_date_newest_first_keeps_existing_rows() { + let path = temp_csv_path("csv-sort-newest-first"); + std::fs::write( + &path, + "#name;username;password;url;max_connections;exp_date\n\ +old;old-user;old-pass;http://old.example;1;2026-01-01 00:00:00\n\ +new;new-user;new-pass;http://new.example;1;2027-01-01 00:00:00\n\ +missing;missing-user;missing-pass;http://missing.example;1;\n", + ) + .expect("write csv fixture"); + + let changed = csv_patch_batch_sort_by_exp_date(InputType::XtreamBatch, &path, AliasExpDateSortOrder::NewestFirst) + .await + .expect("sort succeeds"); + + assert!(changed); + let (_, aliases) = csv_read_inputs_from_path(InputType::XtreamBatch, &path).await.expect("read sorted csv"); + assert_eq!(aliases.len(), 3); + assert_eq!(aliases[0].name.as_ref(), "new"); + assert_eq!(aliases[1].name.as_ref(), "old"); + assert_eq!(aliases[2].name.as_ref(), "missing"); + let _ = std::fs::remove_file(path); + } } diff --git a/backend/src/utils/ffmpeg.rs b/backend/src/utils/ffmpeg.rs index 8ecec1bba..b20f827eb 100644 --- a/backend/src/utils/ffmpeg.rs +++ b/backend/src/utils/ffmpeg.rs @@ -5,8 +5,11 @@ use reqwest::{ Client, StatusCode, }; use serde_json::Value; -use shared::model::MediaQuality; -use shared::utils::{default_thumbnail_height, default_thumbnail_width, is_dash_url, is_hls_url, sanitize_sensitive_info}; +use shared::{ + model::MediaQuality, + utils::{is_dash_url, is_hls_url, sanitize_sensitive_info}, + defaults::{default_thumbnail_height, default_thumbnail_width}, +}; use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; @@ -18,9 +21,9 @@ use tokio::{ }; use url::Url; -const FFMPEG_TIMEOUT: Duration = Duration::from_secs(60); +const FFMPEG_TIMEOUT: Duration = Duration::from_mins(1); const FFPROBE_SEEKABLE_MAX_WINDOW_BYTES: u64 = 32 * 1024 * 1024; -const FFPROBE_TEMP_STALE_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); +const FFPROBE_TEMP_STALE_MAX_AGE: Duration = Duration::from_hours(24); #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProbeFailureKind { @@ -1212,7 +1215,7 @@ mod tests { }; use crate::model::ProxyConfig; use serde_json::json; - use shared::utils::{default_thumbnail_height, default_thumbnail_width}; + use shared::defaults::{default_thumbnail_height, default_thumbnail_width}; use std::{io, path::Path, pin::Pin, task::{Context, Poll}}; use std::sync::{atomic::{AtomicUsize, Ordering}, Arc, Mutex}; use tokio::{ diff --git a/backend/src/utils/file/config_reader.rs b/backend/src/utils/file/config_reader.rs index 9ceafb919..da1f103d1 100644 --- a/backend/src/utils/file/config_reader.rs +++ b/backend/src/utils/file/config_reader.rs @@ -15,13 +15,16 @@ use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; use log::{error, info, warn}; use serde::Serialize; -use shared::error::TuliproxError; -use shared::foundation::prepare_templates; -use shared::model::{ - ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, - MsgKind, PatternTemplate, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, +use shared::{ + error::TuliproxError, + foundation::prepare_templates, + model::{ + ApiProxyConfigDto, AppConfigDto, ConfigDto, ConfigInputAliasDto, ConfigPaths, HdHomeRunDeviceOverview, InputType, + MsgKind, PatternTemplate, SourcesConfigDto, TargetUserDto, TemplateDefinitionDto, + }, + utils::{CONSTANTS, PROVIDER_SCHEME_PREFIX}, + defaults::{generate_default_access_secret, generate_default_encrypt_secret, TEMPLATE_FILE}, }; -use shared::utils::{generate_default_access_secret, generate_default_encrypt_secret, CONSTANTS, PROVIDER_SCHEME_PREFIX, TEMPLATE_FILE}; use std::collections::{HashMap, HashSet}; use std::env; use std::fs::File; diff --git a/backend/src/utils/file/file_lock_manager.rs b/backend/src/utils/file/file_lock_manager.rs index 94a676817..9a930389e 100644 --- a/backend/src/utils/file/file_lock_manager.rs +++ b/backend/src/utils/file/file_lock_manager.rs @@ -175,7 +175,7 @@ pub fn exec_file_lock_prune(app_state: &Arc) { tokio::spawn({ async move { loop { - tokio::time::sleep(Duration::from_secs(60)).await; + tokio::time::sleep(Duration::from_mins(1)).await; app_state.app_config.file_locks.prune_unused_locks().await; } } diff --git a/backend/src/utils/file/file_utils.rs b/backend/src/utils/file/file_utils.rs index f867ac5b2..cc549ddd3 100644 --- a/backend/src/utils/file/file_utils.rs +++ b/backend/src/utils/file/file_utils.rs @@ -1,8 +1,11 @@ use crate::utils::debug_if_enabled; use log::{debug, error, trace}; use path_clean::PathClean; -use shared::error::str_to_io_error; -use shared::utils::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_DIR, DEFAULT_WEB_ROOT_ENV_VAR, MAPPING_FILE, SOURCE_FILE, TEMPLATE_FILE, USER_FILE, USER_GROUP_FILE}; +use shared::{ + error::str_to_io_error, + utils::{DEFAULT_HOME_ENV_VAR, DEFAULT_WEB_ROOT_ENV_VAR, }, + defaults::{API_PROXY_FILE, CONFIG_FILE, CONFIG_PATH, DEFAULT_WEB_DIR, MAPPING_FILE, SOURCE_FILE, TEMPLATE_FILE, USER_FILE, USER_GROUP_FILE}, +}; use std::borrow::Cow; use std::collections::HashSet; use std::fs::{File, OpenOptions}; @@ -149,7 +152,7 @@ fn resolve_config_scoped_path(config_path: &str, candidate: &str) -> String { #[inline] pub fn resolve_mapping_file_path(config_path: &str, mapping_path: Option<&str>) -> String { let configured = mapping_path.map(str::trim).filter(|path| !path.is_empty()).map(ToString::to_string); - let candidate = if shared::utils::is_blank_or_default_mapping_path(&configured) { + let candidate = if shared::defaults::is_blank_or_default_mapping_path(&configured) { get_default_mappings_path(config_path) } else { configured.unwrap_or_else(|| get_default_mappings_path(config_path)) @@ -160,7 +163,7 @@ pub fn resolve_mapping_file_path(config_path: &str, mapping_path: Option<&str>) #[inline] pub fn resolve_template_file_path(config_path: &str, template_path: Option<&str>) -> String { let configured = template_path.map(str::trim).filter(|path| !path.is_empty()).map(ToString::to_string); - let candidate = if shared::utils::is_blank_or_default_template_path(&configured) { + let candidate = if shared::defaults::is_blank_or_default_template_path(&configured) { get_default_templates_path(config_path) } else { configured.unwrap_or_else(|| get_default_templates_path(config_path)) diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 916de24df..75949e5b7 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -18,7 +18,6 @@ use reqwest::{ redirect::Policy, StatusCode, }; -use shared::utils::DEFAULT_USER_AGENT; use shared::{ error::{string_to_io_error, TuliproxError}, model::{format_elapsed_time, InputFetchMethod, OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, @@ -26,6 +25,7 @@ use shared::{ filter_request_header, human_readable_byte_size, sanitize_sensitive_info, CONTENT_TYPE_JSON, ENCODING_DEFLATE, ENCODING_GZIP, }, + defaults::{DEFAULT_USER_AGENT} }; use std::{ collections::{HashMap, HashSet}, @@ -46,6 +46,21 @@ use url::Url; static PROXY_DIAGNOSTICS_ONCE: Once = Once::new(); +#[derive(Debug, Clone, Copy, Default)] +pub struct RequestFetchOptions { + pub attempt_idle_timeout: Option, +} + +impl RequestFetchOptions { + pub fn with_attempt_idle_timeout(timeout: Duration) -> Self { + Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))) } + } + + fn attempt_idle_timeout_or_default(self) -> Duration { + self.attempt_idle_timeout.unwrap_or_else(|| Duration::from_secs(STREAM_IDLE_TIMEOUT)) + } +} + fn log_proxy_diagnostics(config: &Config) { PROXY_DIAGNOSTICS_ONCE.call_once(|| { if let Some(proxy_cfg) = config.proxy.as_ref() { @@ -451,6 +466,158 @@ async fn execute_attempt_request( base_client.execute(request).await } +/// Response returned after applying provider URL failover without applying the generic resource retry policy. +pub struct ProviderFailoverResponse { + pub response: reqwest::Response, + pub provider_url_index: Option, +} + +/// Sends one logical request while allowing one bounded provider URL failover cycle. +/// +/// This reuses provider URL resolution, provider DNS handling, global client/proxy configuration and failover status +/// classification, but deliberately does not apply `reverse_proxy.resource_retry` attempts or backoff. +#[allow(clippy::too_many_lines)] +pub async fn send_with_provider_failover_only( + app_config: &Arc, + url: &Url, + provider: Option<&Arc>, + allow_redirects: bool, + mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + let failover_patterns = app_config + .config + .load() + .reverse_proxy + .as_ref() + .map_or_else(|| ResourceRetryConfig::default().failover_redirect_patterns, |rp| { + rp.resource_retry.failover_redirect_patterns.clone() + }); + + let start_provider_index = provider_start_index(provider); + let mut provider_url_index = start_provider_index; + + 'provider_loop: loop { + let mut attempted_dns_ips = HashSet::new(); + + 'ip_loop: loop { + let attempt_target = resolve_attempt_target_at_provider_index(url, provider, provider_url_index); + if log_enabled!(Level::Debug) { + if let Some(current_provider) = provider { + let attempt_target_log = format_request_target_for_logging(&attempt_target); + debug!( + "Provider '{}' acquiring URL index {} of {}: {}", + current_provider.name, + provider_url_index, + current_provider.urls.len(), + sanitize_sensitive_info(attempt_target_log.as_str()) + ); + } + } + + let request_builder = send(&attempt_target.request_url); + let (base_client, request_result) = request_builder.build_split(); + let mut request = request_result.map_err(|err| { + string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(err.to_string().as_str()))) + })?; + apply_attempt_to_request(&mut request, &attempt_target)?; + + match execute_attempt_request(app_config, base_client, request, &attempt_target).await { + Ok(response) => { + let status = response.status(); + if allow_redirects && status.is_redirection() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + let is_failover = is_failover_redirect(response.url(), &failover_patterns); + if !is_failover && !should_trigger_failover(status) { + if status.is_success() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + let last_provider_failure = format!( + "status {} while trying {}", + format_http_status(status), + sanitize_sensitive_info(attempt_target.request_url.as_str()) + ); + + if let Some(current_provider) = provider { + let reason = format!("status {}", format_http_status(status)); + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + reason.as_str(), + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + Err(err) => { + if (err.is_timeout() || err.is_connect()) + && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) + { + continue 'ip_loop; + } + + let last_provider_failure = format!( + "connection error while trying {}: {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + ); + + if err.is_timeout() || err.is_connect() { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "connection error", + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + } + + return Err(string_to_io_error(format!( + "Request error: {}", + sanitize_sensitive_info(err.to_string().as_str()) + ))); + } + } + } + } +} + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] pub fn calculate_retry_backoff(base_delay_ms: u64, multiplier: f64, attempt: u32) -> u64 { let base = base_delay_ms.max(1); @@ -475,7 +642,16 @@ pub async fn send_with_retry_and_provider( allow_redirects: bool, send: impl FnMut(&Url) -> reqwest::RequestBuilder, ) -> Result { - send_with_retry_and_provider_policy(app_config, url, provider, allow_redirects, true, send).await + send_with_retry_and_provider_policy_with_options( + app_config, + url, + provider, + allow_redirects, + true, + RequestFetchOptions::default(), + send, + ) + .await } /// Canonical retry and provider-failover entry point for outbound resource requests. @@ -495,8 +671,53 @@ pub async fn send_with_retry_and_provider_policy( provider: Option<&Arc>, allow_redirects: bool, retry_enabled: bool, - mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, + send: impl FnMut(&Url) -> reqwest::RequestBuilder, ) -> Result { + send_with_retry_and_provider_policy_with_options( + app_config, + url, + provider, + allow_redirects, + retry_enabled, + RequestFetchOptions::default(), + send, + ) + .await +} + +#[allow(clippy::too_many_lines)] +pub async fn send_with_retry_and_provider_policy_with_options( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + retry_enabled: bool, + options: RequestFetchOptions, + send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + send_with_retry_and_provider_policy_with_options_result( + app_config, + url, + provider, + allow_redirects, + retry_enabled, + options, + send, + ) + .await + .map(|result| result.response) +} + +#[allow(clippy::too_many_lines)] +pub async fn send_with_retry_and_provider_policy_with_options_result( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + retry_enabled: bool, + options: RequestFetchOptions, + mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { let config = app_config.config.load(); let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( || { @@ -511,7 +732,7 @@ pub async fn send_with_retry_and_provider_policy( let max_attempts = if retry_enabled { max_attempts } else { 1 }; drop(config); - let idle_timeout = Duration::from_secs(STREAM_IDLE_TIMEOUT); + let idle_timeout = options.attempt_idle_timeout_or_default(); let idle = sleep(idle_timeout); tokio::pin!(idle); @@ -548,6 +769,9 @@ pub async fn send_with_retry_and_provider_policy( string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(&err.to_string()))) })?; apply_attempt_to_request(&mut request, &attempt_target)?; + if let Some(timeout) = options.attempt_idle_timeout { + *request.timeout_mut() = Some(timeout); + } tokio::select! { () = &mut idle => { @@ -603,14 +827,20 @@ pub async fn send_with_retry_and_provider_policy( if let Some(current_provider) = provider { current_provider.set_current_index(provider_url_index); } - return Ok(response); + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); } let is_failover = is_failover_redirect(response.url(), &failover_patterns); if !is_failover && status.is_success() { if let Some(current_provider) = provider { current_provider.set_current_index(provider_url_index); } - return Ok(response); + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); } last_provider_failure = Some(format!( @@ -753,6 +983,363 @@ pub async fn send_with_retry_and_provider_policy( Err(string_to_io_error("All attempts and providers exhausted")) } +fn prepare_input_request_headers( + app_config: &Arc, + input: &InputSource, + headers: Option<&HeaderMap>, +) -> (HashMap, Option) { + let custom_headers = headers + .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); + + let config = app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let merged = get_request_headers( + Some(&input.headers), + custom_headers.as_ref(), + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + + let request_headers: HashMap = merged + .iter() + .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) + .collect(); + + (request_headers, default_user_agent) +} + +#[allow(clippy::implicit_hasher)] +pub async fn send_input_with_retry_and_provider_policy_with_options_result( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: RequestFetchOptions, +) -> Result { + let (request_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); + send_with_retry_and_provider_policy_with_options_result( + app_config, + url, + input.get_provider(), + false, + true, + options, + |resolved_url| { + get_client_request( + client, + input.method, + Some(&request_headers), + resolved_url, + None, + None, + default_user_agent.as_deref(), + ) + }, + ) + .await +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::implicit_hasher)] +pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + max_redirects: usize, + options: RequestFetchOptions, +) -> Result { + let config = app_config.config.load(); + let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( + || { + let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); + (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) + }, + |rp| { + let (a, b, c) = rp.resource_retry.get_retry_values(); + (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) + }, + ); + drop(config); + + let (base_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); + let provider = input.get_provider(); + let max_provider_attempts = provider.as_ref().map_or(0, |p| p.urls.len()); + let start_provider_index = provider_start_index(provider); + let mut provider_url_index = start_provider_index; + let mut last_provider_failure: Option = None; + let idle_timeout = options.attempt_idle_timeout_or_default(); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + 'provider_loop: loop { + 'attempt_loop: for attempt in 0..max_attempts { + let mut current_url = url.clone(); + let mut current_headers = base_headers.clone(); + let mut remaining_redirects = max_redirects; + let mut attempted_dns_ips = HashSet::new(); + + 'redirect_loop: loop { + let attempt_target = resolve_attempt_target_at_provider_index(¤t_url, provider, provider_url_index); + if log_enabled!(Level::Debug) { + if let Some(current_provider) = provider { + let attempt_target_log = format_request_target_for_logging(&attempt_target); + debug!( + "Provider '{}' attempting URL index {} of {}: {}", + current_provider.name, + provider_url_index, + max_provider_attempts, + sanitize_sensitive_info(attempt_target_log.as_str()) + ); + } + } + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + + let request_builder = get_client_request( + client, + input.method, + Some(¤t_headers), + &attempt_target.request_url, + None, + None, + default_user_agent.as_deref(), + ); + let (base_client, request_result) = request_builder.build_split(); + let mut request = request_result.map_err(|err| { + string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(err.to_string().as_str()))) + })?; + apply_attempt_to_request(&mut request, &attempt_target)?; + if let Some(timeout) = options.attempt_idle_timeout { + *request.timeout_mut() = Some(timeout); + } + + tokio::select! { + () = &mut idle => { + warn!("Request idle for too long: {}", sanitize_sensitive_info(url.as_str())); + last_provider_failure = Some(format!( + "idle timeout while trying {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "idle timeout", + ) { + continue 'provider_loop; + } + if max_provider_attempts > 0 { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("idle timeout"), + ); + } + } + + if attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + warn!("Idle timeout, retrying same URL in {}ms (attempt {})", delay, attempt + 1); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + return Err(string_to_io_error(format!("Request timed out and no retries left: {}", sanitize_sensitive_info(url.as_str())))); + } + + result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { + match result { + Ok(response) => { + if response.status().is_redirection() { + if remaining_redirects == 0 { + return Err(string_to_io_error(format!( + "Too many redirects while requesting {}", + sanitize_sensitive_info(url.as_str()) + ))); + } + + let response_base_url = response.url().clone(); + let Some(location) = response.headers().get(reqwest::header::LOCATION) else { + return Err(string_to_io_error(format!( + "Redirect response missing location header for {}", + sanitize_sensitive_info(current_url.as_str()) + ))); + }; + let Ok(location_str) = location.to_str() else { + return Err(string_to_io_error(format!( + "Redirect response contains invalid location header for {}", + sanitize_sensitive_info(current_url.as_str()) + ))); + }; + let next_url = response_base_url + .join(location_str) + .or_else(|_| Url::parse(location_str)) + .map_err(|_| { + string_to_io_error(format!( + "Redirect response contains invalid location URL for {}", + sanitize_sensitive_info(current_url.as_str()) + )) + })?; + + if !same_origin(&response_base_url, &next_url) { + strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); + } + current_url = next_url; + remaining_redirects = remaining_redirects.saturating_sub(1); + continue 'redirect_loop; + } + + let status = response.status(); + let is_failover = is_failover_redirect(response.url(), &failover_patterns); + if !is_failover && status.is_success() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + last_provider_failure = Some(format!( + "status {} while trying {}", + format_http_status(status), + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + + let provider_failover_exhausted = (is_failover || should_trigger_failover(status)) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if is_failover || should_trigger_failover(status) { + if let Some(current_provider) = provider { + let reason = format!("status {}", format_http_status(status)); + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + reason.as_str(), + ) { + continue 'provider_loop; + } + } + } + + let is_retryable = status.is_server_error() + || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT); + if attempt < max_attempts - 1 && is_retryable { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request failed"), + ); + } + } + + return Err(string_to_io_error(format!( + "Request failed ({}): {}", + format_http_status(status), + sanitize_sensitive_info(url.as_str()) + ))); + } + Err(err) => { + if (err.is_timeout() || err.is_connect()) + && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) + { + continue 'redirect_loop; + } + + last_provider_failure = Some(format!( + "connection error while trying {}: {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + )); + + let provider_failover_exhausted = (err.is_timeout() || err.is_connect()) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if err.is_timeout() || err.is_connect() { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "connection error", + ) { + continue 'provider_loop; + } + } + } + + if (err.is_timeout() || err.is_connect()) && attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request error"), + ); + } + } + + return Err(string_to_io_error(format!( + "Request error: {}", + sanitize_sensitive_info(err.to_string().as_str()) + ))); + } + } + } + } + } + } + + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "retries exhausted for current URL", + ) { + continue 'provider_loop; + } + + if max_provider_attempts > 0 { + let last_failure = last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_failure, + ); + } + } + + break; + } + + Err(string_to_io_error("All attempts and providers exhausted")) +} + fn is_failover_redirect(url: &Url, patterns: &[Arc]) -> bool { let redirect_url = url.as_str(); patterns.iter().any(|pattern| pattern.is_match(redirect_url)) @@ -1241,61 +1828,103 @@ pub async fn get_remote_content_as_stream( headers: Option<&HeaderMap>, url: &Url, ) -> Result<(DynReader, String), Error> { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); - - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let merged = get_request_headers( - Some(&input.headers), - custom_headers.as_ref(), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - - let headers: HashMap = merged - .iter() - .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) - .collect(); - - let response = send_with_retry_and_provider(app_config, url, input.get_provider(), false, |resolved_url| { - get_client_request( - client, - input.method, - Some(&headers), - resolved_url, - None, - None, - default_user_agent.as_deref(), - ) - }) - .await?; + get_remote_content_as_stream_with_options(app_config, client, input, headers, url, RequestFetchOptions::default()) + .await +} +#[allow(clippy::implicit_hasher)] +async fn get_remote_content_as_stream_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: RequestFetchOptions, +) -> Result<(DynReader, String), Error> { + let response = + send_input_with_retry_and_provider_policy_with_options_result(app_config, client, input, headers, url, options) + .await? + .response; let response_url = response.url().to_string(); let reader = build_decoded_stream_reader(response).await?; Ok((reader, response_url)) } -async fn get_remote_content( +async fn read_stream_to_string_with_options( + stream: &mut DynReader, + response_url: &str, + options: RequestFetchOptions, +) -> Result { + let mut content = String::new(); + if let Some(timeout) = options.attempt_idle_timeout { + tokio::time::timeout(timeout, stream.read_to_string(&mut content)) + .await + .map_err(|_| { + Error::new( + ErrorKind::TimedOut, + format!( + "Timed out reading content body: {}", + sanitize_sensitive_info(response_url) + ), + ) + })? + .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; + } else { + stream + .read_to_string(&mut content) + .await + .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; + } + Ok(content) +} + +fn text_body_retry_values(app_config: &Arc, options: RequestFetchOptions) -> (u32, u64, f64) { + if options.attempt_idle_timeout.is_none() { + return (1, 0, 1.0); + } + let config = app_config.config.load(); + let values = config.reverse_proxy.as_ref().map_or_else( + ResourceRetryConfig::get_default_retry_values, + |rp| rp.resource_retry.get_retry_values(), + ); + drop(config); + values +} + +async fn sleep_before_text_body_retry(attempt: u32, backoff_ms: u64, backoff_multiplier: f64, err: &Error) { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + warn!( + "Text response body timed out, retrying in {}ms (attempt {}): {}", + delay, + attempt + 1, + sanitize_sensitive_info(err.to_string().as_str()) + ); + tokio::time::sleep(Duration::from_millis(delay)).await; +} + +async fn get_remote_content_with_options( app_config: &Arc, client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, + options: RequestFetchOptions, ) -> Result<(String, String), Error> { - let (mut stream, response_url) = get_remote_content_as_stream(app_config, client, input, headers, url) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - let mut content = String::new(); - stream - .read_to_string(&mut content) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - Ok((content, response_url)) + let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options); + for attempt in 0..max_attempts { + let (mut stream, response_url) = get_remote_content_as_stream_with_options(app_config, client, input, headers, url, options) + .await + .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; + match read_stream_to_string_with_options(&mut stream, response_url.as_str(), options).await { + Ok(content) => return Ok((content, response_url)), + Err(err) if err.kind() == ErrorKind::TimedOut && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &err).await; + } + Err(err) => return Err(err), + } + } + Err(string_to_io_error("Text response body retry attempts exhausted")) } async fn get_remote_content_with_headers( @@ -1349,98 +1978,41 @@ async fn get_remote_content_with_headers( Ok((content, response_url, response_headers)) } -async fn get_remote_content_with_manual_redirects( +#[allow(clippy::too_many_lines)] +async fn get_remote_content_with_manual_redirects_and_options( app_config: &Arc, client: &reqwest::Client, input: &InputSource, headers: Option<&HeaderMap>, url: &Url, max_redirects: usize, + options: RequestFetchOptions, ) -> Result<(String, String), Error> { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); - - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let merged = get_request_headers( - Some(&input.headers), - custom_headers.as_ref(), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - - let headers: HashMap = merged - .iter() - .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) - .collect(); - - let mut current_url = url.clone(); - let mut current_headers = headers; - let mut remaining_redirects = max_redirects; - loop { - let response = - send_with_retry_and_provider(app_config, ¤t_url, input.get_provider(), true, |resolved_url| { - get_client_request( - client, - input.method, - Some(¤t_headers), - resolved_url, - None, - None, - default_user_agent.as_deref(), - ) - }) - .await?; - let response_base_url = response.url().clone(); - - if response.status().is_redirection() { - if remaining_redirects == 0 { - return Err(string_to_io_error(format!( - "Too many redirects while requesting {}", - sanitize_sensitive_info(url.as_str()) - ))); - } - - let Some(location) = response.headers().get(reqwest::header::LOCATION) else { - return Err(string_to_io_error(format!( - "Redirect response missing location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let Ok(location_str) = location.to_str() else { - return Err(string_to_io_error(format!( - "Redirect response contains invalid location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let next_url = - response_base_url.join(location_str).or_else(|_| Url::parse(location_str)).map_err(|_| { - string_to_io_error(format!( - "Redirect response contains invalid location URL for {}", - sanitize_sensitive_info(current_url.as_str()) - )) - })?; - - if !same_origin(&response_base_url, &next_url) { - strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); - } - current_url = next_url; - remaining_redirects = remaining_redirects.saturating_sub(1); - continue; - } + let (max_body_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options); + for body_attempt in 0..max_body_attempts { + let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config, + client, + input, + headers, + url, + max_redirects, + options, + ) + .await? + .response; let response_url = response.url().to_string(); let mut stream = build_decoded_stream_reader(response).await?; - let mut content = String::new(); - stream - .read_to_string(&mut content) - .await - .map_err(|e| string_to_io_error(format!("Failed to read content: {e}")))?; - return Ok((content, response_url)); + match read_stream_to_string_with_options(&mut stream, response_url.as_str(), options).await { + Ok(content) => return Ok((content, response_url)), + Err(err) if err.kind() == ErrorKind::TimedOut && body_attempt + 1 < max_body_attempts => { + sleep_before_text_body_retry(body_attempt, backoff_ms, backoff_multiplier, &err).await; + } + Err(err) => return Err(err), + } } + Err(string_to_io_error("Text response body retry attempts exhausted")) } async fn get_remote_content_with_manual_redirects_and_headers( @@ -1598,16 +2170,37 @@ pub async fn download_text_content( headers: Option<&HeaderMap>, persist_filepath: Option, trace_log: bool, +) -> Result<(String, String), Error> { + download_text_content_with_options( + app_config, + client, + input, + headers, + persist_filepath, + trace_log, + RequestFetchOptions::default(), + ) + .await +} + +pub async fn download_text_content_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, + options: RequestFetchOptions, ) -> Result<(String, String), Error> { let start_time = tokio::time::Instant::now(); let result = if let Ok(url) = input.url.parse::() { let result = if url.scheme() == "file" { match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|c| (c, url.to_string())), + Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), } } else { - get_remote_content(app_config, client, input, headers, &url).await + get_remote_content_with_options(app_config, client, input, headers, &url, options).await }; match result { Ok((content, response_url)) => { @@ -1682,16 +2275,49 @@ pub async fn download_text_content_with_manual_redirects( persist_filepath: Option, trace_log: bool, max_redirects: usize, +) -> Result<(String, String), Error> { + download_text_content_with_manual_redirects_and_options( + app_config, + client, + input, + headers, + persist_filepath, + trace_log, + max_redirects, + RequestFetchOptions::default(), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn download_text_content_with_manual_redirects_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, + max_redirects: usize, + options: RequestFetchOptions, ) -> Result<(String, String), Error> { let start_time = tokio::time::Instant::now(); let result = if let Ok(url) = input.url.parse::() { let result = if url.scheme() == "file" { match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|c| (c, url.to_string())), + Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), } } else { - get_remote_content_with_manual_redirects(app_config, client, input, headers, &url, max_redirects).await + get_remote_content_with_manual_redirects_and_options( + app_config, + client, + input, + headers, + &url, + max_redirects, + options, + ) + .await }; match result { Ok((content, response_url)) => { @@ -1983,18 +2609,25 @@ mod tests { use super::{ is_safe_cross_origin_redirect_header, next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, - resolve_attempt_target, same_origin, send_with_retry_and_provider, send_with_retry_and_provider_policy, - should_try_next_ip_on_connect_error, strip_sensitive_headers_for_cross_origin_redirect, + resolve_attempt_target, same_origin, send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_with_retry_and_provider, send_with_retry_and_provider_policy, should_try_next_ip_on_connect_error, + strip_sensitive_headers_for_cross_origin_redirect, RequestFetchOptions, }; use crate::{ - model::{AppConfig, Config, ConfigProvider, MediaToolCapabilities, ResourceRetryConfig, ReverseProxyConfig, SourcesConfig}, - utils::{FileLockManager, DEFAULT_USER_AGENT}, + model::{ + AppConfig, Config, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, + ReverseProxyConfig, SourcesConfig, + }, + utils::{FileLockManager}, }; use arc_swap::{ArcSwap, ArcSwapOption}; use shared::model::{ - ConfigPaths, ConfigProviderDto, DnsScheme, OnConnectErrorPolicy, ProviderDnsDto, ProviderUrlSelectionPolicy, + ConfigPaths, ConfigProviderDto, DnsScheme, InputFetchMethod, OnConnectErrorPolicy, ProviderDnsDto, + ProviderUrlSelectionPolicy, }; use shared::utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}; + use shared::defaults::{DEFAULT_USER_AGENT}; + use std::{ collections::{HashMap, HashSet}, net::SocketAddr, @@ -2333,10 +2966,134 @@ mod tests { Ok((addr, accepted, handle)) } + async fn start_plain_http_server_with_response( + response: String, + ) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let accepted = Arc::new(AtomicUsize::new(0)); + let accepted_clone = Arc::clone(&accepted); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + continue; + }; + accepted_clone.fetch_add(1, Ordering::SeqCst); + let response = response.clone(); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let _ = socket.read(&mut buf).await; + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + }); + } + }); + + Ok((addr, accepted, handle)) + } + async fn start_plain_http_server() -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { start_plain_http_server_with_body(b"ok").await } + #[tokio::test] + async fn manual_redirect_provider_failover_restarts_from_provider_entry() { + let (redirect_addr, redirect_hits, redirect_handle) = match start_plain_http_server_with_response( + "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + ) + .await + { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping manual_redirect_provider_failover_restarts_from_provider_entry: {err}"); + return; + } + Err(err) => panic!("failed to start redirect target server: {err}"), + }; + let redirect_url = format!("http://127.0.0.1:{}/redirected", redirect_addr.port()); + let provider_a_response = format!( + "HTTP/1.1 302 Found\r\nLocation: {redirect_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + let provider_entrypoint = start_plain_http_server_with_response(provider_a_response) + .await + .expect("provider a test server should start"); + let successful_mirror = + start_plain_http_server_with_body(b"provider-b").await.expect("provider b test server should start"); + + let mut cfg = Config { + connect_timeout_secs: 1, + ..Config::default() + }; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { + max_attempts: 1, + ..ResourceRetryConfig::default() + }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_millis(400)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![ + format!("http://127.0.0.1:{}", provider_entrypoint.0.port()).into(), + format!("http://127.0.0.1:{}", successful_mirror.0.port()).into(), + ], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let input = InputSource { + name: Arc::::from("test"), + url: "provider://provider-a/live/u/p/1.m3u8".to_string(), + provider: Some(provider), + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::default(), + }; + let entry_url = Url::parse(input.url.as_str()).expect("provider URL should parse"); + + let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &app_config, + &client, + &input, + None, + &entry_url, + 5, + RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)), + ) + .await + .expect("request should fail over from redirected target to next provider entry"); + let provider_url_index = response.provider_url_index; + let body = response.response.text().await.expect("body should be readable"); + + assert_eq!(body, "provider-b"); + assert_eq!(provider_url_index, Some(1)); + assert_eq!(provider_entrypoint.1.load(Ordering::SeqCst), 1); + assert_eq!(redirect_hits.load(Ordering::SeqCst), 1); + assert_eq!(successful_mirror.1.load(Ordering::SeqCst), 1); + + provider_entrypoint.2.abort(); + successful_mirror.2.abort(); + redirect_handle.abort(); + } + #[tokio::test] async fn test_provider_request_chain_starts_from_last_successful_url() { let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { @@ -2369,6 +3126,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }); let app_config = make_test_app_config(cfg); let client = reqwest::Client::builder() @@ -2443,6 +3201,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }); let app_config = make_test_app_config(cfg); let client = reqwest::Client::builder() @@ -2503,6 +3262,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }); let app_config = make_test_app_config(cfg); let client = reqwest::Client::builder() @@ -2586,6 +3346,7 @@ mod tests { geoip: None, stream_history: None, qos_aggregation: None, + hls_cache: None, }); let app_config = make_test_app_config(cfg); let client = reqwest::Client::builder() diff --git a/backend/src/utils/trakt/client.rs b/backend/src/utils/trakt/client.rs index 5213032fc..f8ba3b2e0 100644 --- a/backend/src/utils/trakt/client.rs +++ b/backend/src/utils/trakt/client.rs @@ -3,7 +3,7 @@ use crate::model::{TraktApiConfig, TraktChartConfig, TraktListConfig, TraktListI use log::{debug, info}; use reqwest::header::{HeaderMap, HeaderValue}; use serde::Deserialize; -use shared::{error::TuliproxError, utils::{trim_last_slash, DEFAULT_USER_AGENT, TRAKT_API_KEY}}; +use shared::{error::TuliproxError, utils::{trim_last_slash,}, defaults::{DEFAULT_USER_AGENT, TRAKT_API_KEY}}; const TRAKT_PAGE_LIMIT: u32 = 100; const TRAKT_MAX_PAGES: u32 = 100; diff --git a/bin/build_local.sh b/bin/build_local.sh index fbd74245d..2785d34ab 100755 --- a/bin/build_local.sh +++ b/bin/build_local.sh @@ -17,6 +17,9 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" # # Only Backend + clean # ./build.sh windows --backend-only --clean # +# # Backend with release optimizations but debug symbols +# ./build.sh linux-musl --backend-only --debug +# # # Help # ./build.sh --help # @@ -31,6 +34,7 @@ export RUSTFLAGS="--remap-path-prefix $HOME=~" BUILD_BACKEND=true BUILD_FRONTEND=true DO_CLEAN=false +BUILD_DEBUG_SYMBOLS=false TARGET="" ######################################## @@ -42,7 +46,7 @@ Usage: $SCRIPT_NAME [options] Targets: - linux-musl x86_64-unknown-linux-musl (cross) + linux-musl x86_64-unknown-linux-musl (cargo zigbuild on macOS, cross otherwise) linux-gnu x86_64-unknown-linux-gnu (cargo on Linux, cross otherwise) armv7 armv7-unknown-linux-musleabihf (cross) aarch64 aarch64-unknown-linux-musl (cross) @@ -54,6 +58,7 @@ Options: --frontend-only Build only frontend --backend-only Build only backend --clean cargo clean before build + --debug Keep debug symbols in release backend binary and disable strip -h, --help Show this help Examples: @@ -61,6 +66,7 @@ Examples: $SCRIPT_NAME armv7 --no-frontend $SCRIPT_NAME aarch64 --no-frontend $SCRIPT_NAME windows --backend-only + $SCRIPT_NAME linux-musl --backend-only --debug $SCRIPT_NAME linux-gnu --clean EOF } @@ -93,6 +99,9 @@ for arg in "$@"; do --clean) DO_CLEAN=true ;; + --debug) + BUILD_DEBUG_SYMBOLS=true + ;; -h|--help) usage exit 0 @@ -124,10 +133,28 @@ fi ######################################## if [ "$BUILD_BACKEND" = true ]; then echo "==> Building backend ($TARGET)" + if [ "$BUILD_DEBUG_SYMBOLS" = true ]; then + echo "==> Keeping release debug symbols and disabling strip" + export CARGO_PROFILE_RELEASE_DEBUG=true + export CARGO_PROFILE_RELEASE_STRIP=false + fi case "$TARGET" in linux-musl) - cross build -p tuliprox --release --target x86_64-unknown-linux-musl + if [ "$(uname)" = "Darwin" ]; then + if ! command -v cargo-zigbuild >/dev/null 2>&1; then + echo "❌ cargo-zigbuild not found. Install with: cargo install cargo-zigbuild" + exit 1 + fi + if ! command -v zig >/dev/null 2>&1; then + echo "❌ zig not found. Install with: brew install zig" + exit 1 + fi + rustup target add x86_64-unknown-linux-musl + cargo zigbuild -p tuliprox --release --target x86_64-unknown-linux-musl + else + cross build -p tuliprox --release --target x86_64-unknown-linux-musl + fi ;; linux-gnu) if [ "$(uname)" = "Linux" ]; then diff --git a/bin/build_resources.sh b/bin/build_resources.sh index fcb1e2033..9fa9f789c 100755 --- a/bin/build_resources.sh +++ b/bin/build_resources.sh @@ -40,19 +40,53 @@ for image in "${resources[@]}"; do if [ "$flag_force" = false ] && [ -e "${output}" ]; then echo "Resource ${resource_name} exists, skipping creation" - continue + else + if ! ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ + -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ + -t 10 -shortest \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -crf 23 \ + -x264-params "keyint=30:min-keyint=30:scenecut=0:bframes=0:open_gop=0" \ + -c:a aac -b:a 128k -ac 2 -ar 48000 \ + -mpegts_flags +resend_headers \ + -muxdelay 0 -muxpreload 0 \ + -f mpegts "${output}"; then + echo "ffmpeg failed for resource ${resource_name}" >&2 + exit 1 + fi fi - if ! ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ - -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ - -t 10 -shortest \ - -c:v libx264 -pix_fmt yuv420p -preset veryfast -crf 23 \ - -x264-params "keyint=30:min-keyint=30:scenecut=0:bframes=0:open_gop=0" \ - -c:a aac -b:a 128k -ac 2 -ar 48000 \ - -mpegts_flags +resend_headers \ - -muxdelay 0 -muxpreload 0 \ - -f mpegts "${output}"; then - echo "ffmpeg failed for resource ${resource_name}" >&2 - exit 1 + if [ "${resource_name}" = "panel_api_provisioning" ]; then + hls_playlist="${image%/*}/panel_api_provisioning_hls.m3u8" + hls_segment_pattern="${image%/*}/panel_api_provisioning_hls_%03d.ts" + hls_segments_missing=false + for index in 0 1 2 3 4 5; do + if [ ! -e "$(printf "%s/panel_api_provisioning_hls_%03d.ts" "${image%/*}" "${index}")" ]; then + hls_segments_missing=true + break + fi + done + if [ "$flag_force" = true ] || [ "$hls_segments_missing" = true ]; then + rm -f "${image%/*}"/panel_api_provisioning_hls_*.ts "${hls_playlist}" + if ! ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ + -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ + -t 12 -shortest \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -crf 23 \ + -g 60 -keyint_min 60 \ + -force_key_frames "expr:gte(t,n_forced*2)" \ + -x264-params "scenecut=0:bframes=0:open_gop=0" \ + -c:a aac -b:a 128k -ac 2 -ar 48000 \ + -mpegts_flags +resend_headers \ + -muxdelay 0 -muxpreload 0 \ + -f hls \ + -hls_time 2 \ + -hls_list_size 0 \ + -hls_segment_type mpegts \ + -hls_segment_filename "${hls_segment_pattern}" \ + "${hls_playlist}"; then + echo "ffmpeg failed for HLS provisioning resource ${resource_name}" >&2 + exit 1 + fi + rm -f "${hls_playlist}" + fi fi done diff --git a/config/source.yml b/config/source.yml index 118dcaf74..2812cc5c9 100644 --- a/config/source.yml +++ b/config/source.yml @@ -24,7 +24,11 @@ sources: username: local device: hdhr1 filter: "!final_channel_lineup!" - options: { ignore_logo: false, share_live_streams: true } + options: + ignore_logo: false + share_live_streams: + hls: true + mpeg_ts: true mapping: - iptv-org - name: iptv-trakt-example @@ -56,4 +60,8 @@ sources: content_type: "both" fuzzy_match_threshold: 80 filter: "!final_channel_lineup!" - options: { ignore_logo: false, share_live_streams: true } + options: + ignore_logo: false + share_live_streams: + hls: true + mpeg_ts: true diff --git a/docker/Dockerfile b/docker/Dockerfile index 1924b8506..07b2f4354 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -103,6 +103,7 @@ COPY resources ./resources RUN set -eu; \ find ./resources -maxdepth 1 -type f -name '*.jpg' | sort | while IFS= read -r image; do \ output="${image%.jpg}.ts"; \ + resource_name="$(basename "${image%.jpg}")"; \ ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ -t 10 -shortest \ @@ -112,6 +113,26 @@ RUN set -eu; \ -mpegts_flags +resend_headers \ -muxdelay 0 -muxpreload 0 \ -f mpegts "${output}" || exit 1; \ + if [ "${resource_name}" = "panel_api_provisioning" ]; then \ + rm -f ./resources/panel_api_provisioning_hls_*.ts ./resources/panel_api_provisioning_hls.m3u8; \ + ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ + -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ + -t 12 -shortest \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -crf 23 \ + -g 60 -keyint_min 60 \ + -force_key_frames "expr:gte(t,n_forced*2)" \ + -x264-params "scenecut=0:bframes=0:open_gop=0" \ + -c:a aac -b:a 128k -ac 2 -ar 48000 \ + -mpegts_flags +resend_headers \ + -muxdelay 0 -muxpreload 0 \ + -f hls \ + -hls_time 2 \ + -hls_list_size 0 \ + -hls_segment_type mpegts \ + -hls_segment_filename "./resources/panel_api_provisioning_hls_%03d.ts" \ + ./resources/panel_api_provisioning_hls.m3u8 || exit 1; \ + rm -f ./resources/panel_api_provisioning_hls.m3u8; \ + fi; \ done # ----------------------------------------------------------------- diff --git a/docker/Dockerfile-cross b/docker/cross.Dockerfile similarity index 88% rename from docker/Dockerfile-cross rename to docker/cross.Dockerfile index 101713ca7..d3a3771bd 100644 --- a/docker/Dockerfile-cross +++ b/docker/cross.Dockerfile @@ -107,6 +107,7 @@ COPY resources ./resources RUN set -eu; \ find ./resources -maxdepth 1 -type f -name '*.jpg' | sort | while IFS= read -r image; do \ output="${image%.jpg}.ts"; \ + resource_name="$(basename "${image%.jpg}")"; \ ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ -t 10 -shortest \ @@ -116,6 +117,26 @@ RUN set -eu; \ -mpegts_flags +resend_headers \ -muxdelay 0 -muxpreload 0 \ -f mpegts "${output}" || exit 1; \ + if [ "${resource_name}" = "panel_api_provisioning" ]; then \ + rm -f ./resources/panel_api_provisioning_hls_*.ts ./resources/panel_api_provisioning_hls.m3u8; \ + ffmpeg -y -nostdin -loop 1 -framerate 30 -i "${image}" \ + -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 \ + -t 12 -shortest \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -crf 23 \ + -g 60 -keyint_min 60 \ + -force_key_frames "expr:gte(t,n_forced*2)" \ + -x264-params "scenecut=0:bframes=0:open_gop=0" \ + -c:a aac -b:a 128k -ac 2 -ar 48000 \ + -mpegts_flags +resend_headers \ + -muxdelay 0 -muxpreload 0 \ + -f hls \ + -hls_time 2 \ + -hls_list_size 0 \ + -hls_segment_type mpegts \ + -hls_segment_filename "./resources/panel_api_provisioning_hls_%03d.ts" \ + ./resources/panel_api_provisioning_hls.m3u8 || exit 1; \ + rm -f ./resources/panel_api_provisioning_hls.m3u8; \ + fi; \ done # ----------------------------------------------------------------- diff --git a/docker/Dockerfile-manual b/docker/manual.Dockerfile similarity index 100% rename from docker/Dockerfile-manual rename to docker/manual.Dockerfile diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index cd56a0823..f1b5966ab 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,6 +8,11 @@ - [Configuration & Setup (Overview)](./configuration/overview.md) - [config.yml (Core System)](./configuration/config.md) - [Streaming & Proxy Behavior](./configuration/reverse-proxy.md) + - [Shared HLS Sessions](./configuration/shared-hls-sessions.md) + - [Shared HLS Configuration](./configuration/shared-hls-configuration.md) + - [Shared HLS Runtime Flow](./configuration/shared-hls-runtime-flow.md) + - [Shared HLS Troubleshooting](./configuration/shared-hls-troubleshooting.md) + - [HLS Cache State Machines](./configuration/hls-cache-state-machine.md) - [Connection Handling](./configuration/connection-handling.md) - [Connection Handling Runtime Flow](./configuration/connection-handling-runtime-flow.md) - [Priorities, Soft Connections and Preemption](./configuration/connection-handling-priorities-and-preemption.md) diff --git a/docs/src/configuration/config.md b/docs/src/configuration/config.md index ec2bb98df..8e5d38141 100644 --- a/docs/src/configuration/config.md +++ b/docs/src/configuration/config.md @@ -64,7 +64,7 @@ metadata_update: | `user_access_control` | Bool | No | `false` | **Security:** If `true`, Tuliprox actively enforces `status` (Active/Banned), `exp_date`, and `max_connections` constraints for users defined in `api-proxy.yml`. If false, those fields are ignored. | | `custom_stream_response_path` | String | No | `null` | Directory path where Tuliprox looks for custom fallback `.ts` files. See section [Custom Stream Response](#custom-stream-responses-fallback-videos) for exact filenames. | | `custom_stream_response_timeout_secs` | Int | No | `0` | Hard timeout (in seconds) that forces the fallback video stream to terminate to prevent infinite bandwidth usage. `0` means endless loop. | -| `custom_stream_response_enabled` | Bool | No | `true` | When `true` (default), the 6 fallback custom-video factories (`channel_unavailable`, `user_connections_exhausted`, `provider_connections_exhausted`, `low_priority_preempted`, `user_account_expired`, `panel_api_provisioning`) serve the configured MPEG-TS video. When `false`, the factories skip the video and the call sites return `custom_stream_response_error_status` instead of an infinite 200 OK loop. Use this behind a reverse proxy with `proxy_intercept_errors on;` to allow dead channels to be severed instead of pinning sockets open. | +| `custom_stream_response_enabled` | Bool | No | `true` | Enables configured custom MPEG-TS fallback videos for stream errors. When `false`, factories skip the video body and call sites return `custom_stream_response_error_status`; useful behind reverse proxies that intercept 4xx/5xx responses instead of keeping fallback sockets open. | | `custom_stream_response_error_status` | Int | No | `502` | HTTP status code returned when `custom_stream_response_enabled` is `false`. Must be a 4xx or 5xx code (the `prepare()` step rejects anything outside that range; `0` is silently clamped to `502`). Common choices: `404` (channel not found), `502` (bad gateway — upstream failed), `503` (service unavailable — overloaded). | --- @@ -844,6 +844,7 @@ custom_stream_response_timeout_secs: 20 * `low_priority_preempted.ts` (User was kicked by an Admin with higher priority) * `user_account_expired.ts` (User's `exp_date` reached) * `panel_api_provisioning.ts` (Loops while a new Provider Account is generated via Panel API) +* `hls_session_or_lease_expired.ts` (Shared-HLS session or access lease expired; restart stream) > **Note**: These Video files are all available in the docker image. diff --git a/docs/src/configuration/connection-handling.md b/docs/src/configuration/connection-handling.md index 79de41f48..7188b0efd 100644 --- a/docs/src/configuration/connection-handling.md +++ b/docs/src/configuration/connection-handling.md @@ -306,7 +306,7 @@ If connection handling looks "strange" for your users, check these first: - Are `admission_strategies` configured? - Is `grace_period_millis` set sensibly? - Are `hls_session_ttl_secs` and `catchup_session_ttl_secs` appropriate for the player in use? -- Is `share_live_streams` enabled and are multiple users watching the same channel? +- Is `share_live_streams.mpeg_ts` enabled and are multiple users watching the same channel? - Is the actual provider already at capacity? ## Recommended reading order diff --git a/docs/src/configuration/hls-cache-state-machine.md b/docs/src/configuration/hls-cache-state-machine.md new file mode 100644 index 000000000..282b53590 --- /dev/null +++ b/docs/src/configuration/hls-cache-state-machine.md @@ -0,0 +1,292 @@ +# HLS Cache State Machines + +This page is the technical reference for the Live HLS cache proxy runtime states. It focuses on the shared `HlsSession`, +the per-playback `HlsAccessLease`, and the timing rules that connect them. + +For an operator-friendly introduction, see [Shared HLS Sessions](./shared-hls-sessions.md). For configuration, see +[Shared HLS Configuration](./shared-hls-configuration.md). + +## Contents + +- [Identifiers](#identifiers) +- [High-level flow](#high-level-flow) +- [HlsAccessLease state machine](#hlsaccesslease-state-machine) +- [HlsSession state model](#hlssession-state-model) +- [Origin account protection](#origin-account-protection) +- [Manifest commit and recovery](#manifest-commit-and-recovery) +- [Segment, MAP, and transient resource delivery](#segment-map-and-transient-resource-delivery) +- [Cleanup rules](#cleanup-rules) +- [Operational timings](#operational-timings) +- [Implementation map](#implementation-map) + +## Identifiers + +| Identifier | Scope | Meaning | +| :--- | :--- | :--- | +| `HlsSessionKey` | Shared content | Stable tuple of `input_id`, HLS kind, and `stream_ref`. It does not include the origin URL, provider URL, username, or password. | +| `proxy_session_id` | Shared public URL identity | Opaque token derived from `HlsSessionKey` and the configured secret. It identifies the shared content session in canonical URLs. | +| `HlsPlaybackFamilyKey` | User/client family | Tuple of Tuliprox username and client fingerprint key. It groups playback attempts by user and client. | +| `hls_access_lease_id` | Per playback URL identity | Random lookup key for one server-side `HlsAccessLease`. It is not a shared-content identity. | +| HLS cache user session token | Per playback admission | Internal Tuliprox user-session token associated with an access lease. | + +The public canonical paths are: + +```text +/hls/shared/live///manifest.m3u8 +/hls/shared/live/// +/hls/shared/live///map/ +/hls/shared/live///r/ +``` + +## High-level flow + +The entry request creates a new playback-specific access lease and redirects the player to the canonical Shared HLS +manifest. The shared content session is created or reused by the canonical manifest request after access-lease validation +and admission checks have passed. + +```mermaid +flowchart TD + A["Entry HLS request"] --> B["Authenticate user and resolve target/input/stream"] + B --> C{"Global hls_cache and target share_live_streams.hls?"} + C -->|no| D["Legacy HLS reverse proxy path"] + C -->|yes| E["Create new Pending HlsAccessLease"] + E --> F["307 redirect to canonical manifest"] + F --> G["Validate access lease and restore context"] + G --> H["Create or reuse shared HlsSession"] + H --> I["Fetch, accept, commit, and render manifest"] + I --> J["Segment, MAP, or transient resource request"] + J --> K["Activate or refresh access lease"] + K --> L["Serve cache hit, demand fetch, prefetch, or transient response"] +``` + +Entry redirects intentionally create distinct access leases for distinct playback starts. A later entry request for the +same user, fingerprint, and channel does not reuse an already activated lease, and existing pending leases are not used as +the identity for a new playback. + +## HlsAccessLease state machine + +`HlsAccessLeaseId` is only the URL lookup key. The lifecycle state lives in `HlsAccessLeaseState`. + +```mermaid +stateDiagram-v2 + [*] --> Pending: Entry redirect creates lease + + Pending --> Pending: Manifest touch + Pending --> Activated: Segment, MAP, or /r resource request + Pending --> Denied: Admission exhausted + Pending --> Expired: Pending deadline or validity boundary reached + + Activated --> Activated: Resource request or active manifest touch + Activated --> Idle: active_until <= now + Activated --> Expired: valid_until <= now + Activated --> Denied: Admission exhausted + + Idle --> Idle: Manifest touch extends valid_until + Idle --> Activated: Later resource request + Idle --> Expired: valid_until <= now + Idle --> Denied: Admission exhausted + + Expired --> Removed: Lifecycle or stale lookup cleanup + Denied --> Removed: Lifecycle cleanup + Removed --> [*] +``` + +### Access lease transitions + +| From | To | Condition | +| :--- | :--- | :--- | +| none | `Pending` | Entry request prepares a new access lease and redirects to the canonical manifest. | +| `Pending` | `Pending` | Canonical manifest validation succeeds. Manifest access alone does not activate the lease. | +| `Pending` | `Activated` | Segment, MAP, or transient `/r` resource request validates with resource access. | +| `Pending` | `Expired` | The pending deadline or validity boundary is reached. The boundary is inclusive. | +| `Pending` | `Denied` | Admission returns exhausted for the bound Tuliprox user session. | +| `Activated` | `Activated` | Media access or an active manifest touch slides the active and valid windows. | +| `Activated` | `Idle` | `active_until_ms <= now_ms` while `valid_until_ms > now_ms`. The bound stream reservation is released. | +| `Activated` | `Expired` | `valid_until_ms <= now_ms`. Cleanup releases the stream reservation if it is still active. | +| `Idle` | `Idle` | A valid manifest request extends `valid_until_ms` without restarting media activity. | +| `Idle` | `Activated` | A later segment, MAP, or transient resource request reactivates the lease. | +| `Idle` | `Expired` | `valid_until_ms <= now_ms`. | +| any usable state | `Denied` | Admission rejects the underlying user session as exhausted. | +| `Expired` or `Denied` | removed | Lifecycle processing, stale lookup, cache path reset, or session cleanup removes the entry. | + +### Lease effects on origin work + +| Lease state | Can validate own URL? | Counts as active viewer? | Contributes effective origin policy? | +| :--- | :---: | :---: | :---: | +| `Pending` | Yes | No | Yes | +| `Activated` | Yes | Yes | Yes | +| `Idle` | Yes | No | No | +| `Expired` | No | No | No | +| `Denied` | No | No | No | + +A session snapshot counts `Activated` leases as active. It uses `Pending` and `Activated` leases to derive the effective +origin acquire policy. `Idle` leases can still recover their own playback path if a later media request arrives before +validity expires, but they do not keep prefetch work active. + +## HlsSession state model + +`HlsSession` is not represented by one single lifecycle enum. Its effective state is composed from several layers. + +| Layer | Stored as | Meaning | +| :--- | :--- | :--- | +| Store presence | HLS session store indexes | Whether the shared session exists. | +| Session identity | `HlsSessionKey` and `proxy_session_id` | Stable shared-content identity. | +| Media mode | `HlsSessionMode` | Normal cache timeline or transient passthrough. | +| Activity | `HlsSessionActivity` | Last authorized manifest/media access, active lease count, active origin work count, and work generation. | +| Origin binding | Origin account binding state | Provider account/session owner binding for upstream work. | +| Cache state | Segment, MAP, transient object maps | Ready, fetching, failed, and temporary object state. | +| GC flag | `gc_marked_for_removal` | Guard used while session cleanup is in progress. | + +```mermaid +stateDiagram-v2 + [*] --> Missing + Missing --> NormalCacheTimeline: First accepted canonical manifest + NormalCacheTimeline --> NormalCacheTimeline: Normal origin manifest commit + NormalCacheTimeline --> TransientPassthrough: Origin manifest requires transient handling + TransientPassthrough --> TransientPassthrough: Transient manifest commit + + NormalCacheTimeline --> Removing: Idle cleanup conditions hold + TransientPassthrough --> Removing: Idle cleanup conditions hold + Removing --> Missing: Store entry and cache directory removed + Removing --> NormalCacheTimeline: Cleanup deferred for active work + Removing --> TransientPassthrough: Cleanup deferred for active work +``` + +### Session modes + +| Mode | Meaning | +| :--- | :--- | +| `NormalCacheTimeline` | Tuliprox parses the origin manifest into a shared timeline and serves segment/MAP URLs through the HLS cache. | +| `TransientPassthrough` | Tuliprox detected a manifest feature that needs controlled transient resource handling, such as certain key resources or unsupported tags. | + +## Origin account protection + +Origin account protection is derived from the most recent successful media response. Manifest-only activity is not enough +to mark a session as media-active. + +```mermaid +stateDiagram-v2 + [*] --> NoMediaYet: No successful media response yet + NoMediaYet --> HardActive: Segment, MAP, or /r response succeeds + HardActive --> SoftActive: now > last_media + target_duration + SoftActive --> Expired: soft overlap window elapsed + SoftActive --> HardActive: New media response succeeds + Expired --> HardActive: New media response succeeds after reacquire +``` + +| Protection state | Condition | +| :--- | :--- | +| `NoMediaYet` | No segment, MAP, or transient resource response has succeeded yet. | +| `HardActive` | Current time is inside the hard-active window after the last authorized media access. | +| `SoftActive` | Hard-active protection elapsed, but the soft overlap window has not elapsed. | +| `Expired` | The soft overlap window elapsed. | + +The timing uses the parsed `#EXT-X-TARGETDURATION` when known. If no target duration is available, Tuliprox uses a +15-second fallback. + +Hard-active sessions are not soft-overlap candidates. Soft-active sessions may be displaced speculatively only when the +runtime account-capacity and cooldown checks allow it. If the original owner returns during the reclaim window, it can +reclaim the binding and the account enters an overlap cooldown. + +## Manifest commit and recovery + +After access-lease validation, the canonical manifest path may fetch an origin manifest and decide whether it can advance +the shared session. + +The manifest commit policy tracks multiple signals: + +| Signal | Purpose | +| :--- | :--- | +| Media sequence and visible segment range | Avoids regressive manifests and invalid jumps. | +| Effective origin host | Avoids unsafe host switches while allowing controlled redirect host pinning. | +| Target duration and segment durations | Drives refresh timing, lease active windows, and startup behavior. | +| Failure counters | Separates temporary origin failures from hard failures that require a fresh commit. | +| Recovery requirements | Forces fresh manifest decisions during cold start, expired revalidation, hard failures, and provisioning handoff. | + +A successful commit renders a visible manifest window and queues prefetch work up to `max_segments_prefetch`, subject to +per-session and global fetch limits. + +Manifest recovery burst is configured by `hls_cache.manifest_recovery_burst.level`. Keep it `off` unless logs show that +manifest recovery needs additional pressure. + +## Segment, MAP, and transient resource delivery + +### Segment objects + +A segment request validates the access lease and then looks up the segment object in the shared session timeline. + +| Object state | Typical response | +| :--- | :--- | +| Ready | Serve from cache, including supported range responses. | +| Known but not ready | Wait briefly or return `503` with `Retry-After`. | +| Missing from session timeline | Return an unavailable/expired response depending on context. | +| Temporary failures below threshold | Keep retrying according to internal retry behavior. | +| Temporary failure threshold reached | Mark usable access leases channel-unavailable. | +| Permanent failure | Mark affected leases channel-unavailable. | + +### MAP objects + +MAP resources use the shared session MAP table and the same access-lease validation model. They are cached and protected +similarly to segments. + +### Transient resources + +Transient resources are exposed through `/r/` URLs. They are request-controlled and access-lease protected. +They are used when the origin manifest contains resources that should not be modeled as normal cached timeline objects. + +## Cleanup rules + +Access lease cleanup and session cleanup are separate. + +An access lease is removed when it expires, is denied, is stale during lookup, or is removed as part of broader session or +runtime cleanup. + +A shared session can be removed only when the idle timeout has elapsed and the session has no active work that would make +cleanup unsafe. + +Cleanup must wait for: + +- active access leases; +- active origin work; +- active origin manifest refresh; +- active segment fetches; +- active MAP fetches; +- prefetch queue entries; +- ready object readers; +- fetching objects; +- active transient resource readers; +- temporary files that still need cleanup. + +Changing the configured HLS cache path clears HLS runtime state because existing sessions and cache object handles point at +the old location. + +## Operational timings + +| Timing | Value or source | Purpose | +| :--- | :--- | :--- | +| Initial pending bootstrap window | 90 seconds | Allows first useful manifest decision during cold start or required fresh commit. | +| Pending follow-up window | `max(10 seconds, 2 * target_duration)` | Shortens a pending lease after a manifest response. | +| Access lease active window | `2 * target_duration` | Keeps a media-active lease active after resource access. | +| Access lease valid window | `hls_cache.session_idle_timeout` | Keeps the lease valid before expiry. Default: 300 seconds. | +| Target duration fallback | 15 seconds | Used before an origin target duration is known. | +| HLS cache GC interval | 30 seconds | Periodic cleanup cadence for cache objects and stale runtime state. | +| Temporary file retention | 30 seconds | Retention window for temporary HLS cache files. | +| Failed segment retention | 10 seconds | Short retention for failed segment state. | +| Origin manifest timeout | `hls_cache.origin_manifest_timeout_ms` | Upstream manifest fetch timeout. Default: 3000 ms. | +| Origin segment timeout | `hls_cache.origin_segment_timeout_ms` | Upstream segment fetch timeout. Default: 10000 ms. | + +## Implementation map + +| Area | Main implementation files | +| :--- | :--- | +| Public routes and request handling | `backend/src/api/endpoints/hls_api.rs` | +| Access lease model and store | `backend/src/api/model/hls_cache/lease.rs` | +| Shared session model | `backend/src/api/model/hls_cache/session.rs` | +| Session manager, lifecycle, runtime config | `backend/src/api/model/hls_cache/manager.rs` | +| Session identity | `backend/src/api/model/hls_cache/ids.rs` | +| Origin account binding/protection | `backend/src/api/model/hls_cache/origin.rs` | +| Manifest refresh and render | `backend/src/api/model/hls_cache/refresh.rs`, `renderer.rs`, `timeline.rs` | +| Segment and MAP fetching | `segment_fetcher.rs`, `map_fetcher.rs` | +| Transient resource handling | `transient.rs` | +| Cache and cleanup | `cache.rs`, `gc.rs` | +| Config DTO and validation | `shared/src/model/config/reverse_proxy.rs`, `shared/src/model/config/target.rs` | diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md index b46394609..13413e0f1 100644 --- a/docs/src/configuration/reverse-proxy.md +++ b/docs/src/configuration/reverse-proxy.md @@ -41,6 +41,7 @@ reverse_proxy: rewrite_secret: A1B2C3D4E5F60718293A4B5C6D7E8F90 stream: cache: + hls_cache: rate_limit: disabled_header: resource_retry: @@ -151,7 +152,7 @@ Stream-type provider behavior: `buffer.size` is defined in chunks of **8192 bytes (8 KB)**. * A size of `1024` equals approx. **8 MB** of RAM per active stream. -* **Shared Streams Impact:** If `share_live_streams` is enabled, each channel consumes at least **12 MB** regardless of client count. +* **Shared Streams Impact:** If `share_live_streams.mpeg_ts` is enabled, each channel consumes at least **12 MB** regardless of client count. Increasing `size` above 1024 (e.g., 2048) increases this to **24 MB** per shared channel. ### 1.2 `throttle_kbps` @@ -226,6 +227,82 @@ reverse_proxy: --- +## 2.1 HLS Cache (`hls_cache`) + +This block configures the Live HLS cache proxy. It only defines operating parameters. +For an operator-friendly explanation, start with [Shared HLS Sessions](./shared-hls-sessions.md). +For the configuration reference, see [Shared HLS Configuration](./shared-hls-configuration.md). +For the shared session, access lease, and transient delivery state machines, see +[HLS Cache State Machines](./hls-cache-state-machine.md). + +```yaml +reverse_proxy: + hls_cache: + cache_path: "/tmp/tuliprox/cache/hls" + strip: + mode: "segments" + value: 0 + cache_duration: 300 + cache_bytes: "10GB" + cache_bytes_per_session: "512MB" + max_segments_prefetch: 6 + max_concurrent_segment_fetches_per_session: 2 + max_concurrent_segment_fetches_global: 64 + origin_manifest_timeout_ms: 3000 + origin_segment_timeout_ms: 10000 + session_idle_timeout: 300 + segment_repair: + max_level: "off" + apply_to_first_segments: 1 + max_parallel_repairs: 1 + postprocess_timeout_ms: 2000 + corrupt_segment_watchdog: + mode: "off" + max_parallel_jobs: 1 +``` + +### HLS Cache Parameter Details + +| Parameter | Type | Default | Technical Impact | +| :--- | :--- | :--- | :--- | +| `cache_path` | Path | `/tmp/tuliprox/cache/hls` | Root directory for future HLS segment and MAP cache objects. | +| `strip.mode` | String | `segments` | Interprets `strip.value` as either a segment count (`segments`) or accumulated `#EXTINF` duration (`seconds`). | +| `strip.value` | Int | `0` | Initial tail holdback for the first rendered HLS view. | +| `cache_duration` | Seconds | `300` | Retention baseline for unprotected HLS cache objects. | +| `cache_bytes` | Byte size | `10GB` | Global HLS cache byte budget. | +| `cache_bytes_per_session` | Byte size | `512MB` | Per-session HLS cache byte budget. | +| `max_segments_prefetch` | Int | `6` | Maximum session-local segment prefetch queue depth. | +| `max_concurrent_segment_fetches_per_session` | Int | `2` | Maximum concurrent future segment fetches for one HLS session. | +| `max_concurrent_segment_fetches_global` | Int | `64` | Maximum concurrent future segment fetches across all HLS sessions. | +| `origin_manifest_timeout_ms` | Milliseconds | `3000` | Timeout for future Origin manifest fetches. | +| `origin_segment_timeout_ms` | Milliseconds | `10000` | Timeout for future Origin segment fetches. | +| `session_idle_timeout` | Seconds | `300` | Idle timeout before a future HLS cache session may be collected. | +| `segment_repair.max_level` | String | `off` | Maximum repair level allowed by the codec-aware MPEG-TS segment repair policy (`off`, `low`, `medium`, `high`). | +| `segment_repair.apply_to_first_segments` | Int | `1` | Number of visible TS objects checked per access-lease activation. | +| `segment_repair.max_parallel_repairs` | Int | `1` | Maximum concurrent repair jobs. Must not exceed `max_segments_prefetch` when repair is enabled. | +| `segment_repair.postprocess_timeout_ms` | Milliseconds | `2000` | Shared timeout for the complete segment post-processing chain, including repair and watchdog work. | +| `segment_repair.corrupt_segment_watchdog.mode` | String | `off` | Optional watchdog for residual TS packet-corrupt warnings after regular repair (`off`, `detect_only`, `sanitize`, `diagnostic`). | +| `segment_repair.corrupt_segment_watchdog.max_parallel_jobs` | Int | `1` | Maximum concurrent watchdog sanitize jobs. | + +Supported byte-size units: + +* `B` +* `KB`, `MB`, `GB`, `TB` as decimal 1000-based units +* `KiB`, `MiB`, `GiB`, `TiB` as binary 1024-based units +* no suffix means bytes + +Important boundaries: + +* `reverse_proxy.hls_cache` only prepares the global cache engine. A target must also set + `options.share_live_streams.hls: true` in `source.yml` before generated HLS live entries use the shared HLS path. +* HLS cache retry behavior is fixed internally and is not user-configurable. +* `reverse_proxy.rewrite_secret` must stay stable. It is used for future HLS `proxy_session_id` values and transient resource IDs. +* `session_idle_timeout` controls HLS cache access-lease validity and idle cleanup. It is independent from + `reverse_proxy.stream.hls_session_ttl_secs`, which belongs to the non-cache HLS request continuity path. +* This block does not enable legacy resource caching; image/logo/EPG caching remains controlled by `reverse_proxy.cache`. + +--- + ## 3. Rate Limiting (`rate_limit`) This block implements an IP-based **Token-Bucket** rate limiter. @@ -484,7 +561,7 @@ When Tuliprox operates as a reverse proxy, it can securely proxy upstream archiv ### Shared Live Streams -Tuliprox can share a live stream (`share_live_streams: true` in the target options of `source.yml`). +Tuliprox can share a live stream (`share_live_streams.mpeg_ts: true` in the target options of `source.yml`). If 5 users watch the same Live-TV channel, Tuliprox pulls the stream only 1x from the provider and multicasts the bytes locally to 5 clients. To ensure a user who tunes in 10 seconds later doesn't get player errors due to missing I-Frames/Keyframes, diff --git a/docs/src/configuration/shared-hls-configuration.md b/docs/src/configuration/shared-hls-configuration.md new file mode 100644 index 000000000..11c2f1ec9 --- /dev/null +++ b/docs/src/configuration/shared-hls-configuration.md @@ -0,0 +1,248 @@ +# Shared HLS Configuration + +Shared HLS requires one global reverse-proxy cache block and one per-target switch. + +The global block tells Tuliprox how to store and manage HLS cache data. The target switch tells Tuliprox which generated +live HLS entries should use the shared HLS cache path. + +## Required switches + +`config.yml`: + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" +``` + +`source.yml`: + +```yaml +targets: + - name: xc_m3u + options: + share_live_streams: + hls: true + mpeg_ts: false +``` + +Shared HLS is enabled for a generated target only when both conditions are true: + +| Condition | File | Required value | +| :--- | :--- | :--- | +| HLS cache engine exists | `config.yml` | `reverse_proxy.hls_cache` is present. | +| Target allows HLS sharing | `source.yml` | `target.options.share_live_streams.hls: true`. | + +If `reverse_proxy.hls_cache` is missing, Tuliprox does not use the shared HLS cache path even when the target option is +set. If the target option is missing or false, that target does not use Shared HLS even when the global cache engine is +configured. + +## Full example + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" + strip: + mode: "segments" + value: 0 + cache_duration: 300 + cache_bytes: "10GB" + cache_bytes_per_session: "512MB" + max_segments_prefetch: 6 + max_concurrent_segment_fetches_per_session: 2 + max_concurrent_segment_fetches_global: 64 + origin_manifest_timeout_ms: 3000 + origin_segment_timeout_ms: 10000 + session_idle_timeout: 300 + manifest_recovery_burst: + level: "off" + segment_repair: + max_level: "off" + apply_to_first_segments: 1 + max_parallel_repairs: 1 + postprocess_timeout_ms: 2000 + size_increase: + low_percent: 2 + medium_percent: 5 + high_percent: 20 + corrupt_segment_watchdog: + mode: "off" + max_parallel_jobs: 1 +``` + +## Target options + +```yaml +targets: + - name: xc_m3u + output: + - type: xtream + - type: m3u + options: + share_live_streams: + hls: true + mpeg_ts: false +``` + +| Option | Default | Meaning | +| :--- | :--- | :--- | +| `share_live_streams.hls` | `false` | Enables Shared HLS for live HLS entries generated by this target. Requires `reverse_proxy.hls_cache`. | +| `share_live_streams.mpeg_ts` | `false` | Enables the separate MPEG-TS live stream sharing path. This is independent from Shared HLS. | + +The object form is recommended. The legacy boolean remains compatible and enables both sharing modes: + +```yaml +share_live_streams: true +``` + +The two switches are separate because HLS and MPEG-TS sharing have different runtime behavior and resource costs. + +## HLS cache parameters + +| Parameter | Default | Recommended first value | Effect | +| :--- | :--- | :--- | :--- | +| `cache_path` | `/tmp/tuliprox/cache/hls` | Persistent writable path, for example `/var/lib/tuliprox/cache/hls` | Root directory for HLS segment and MAP cache objects. | +| `strip.mode` | `segments` | `segments` | Interprets `strip.value` as either a number of segments or a number of seconds. | +| `strip.value` | `0` | `0` | Initial holdback for the first rendered startup view. Increase only when players start too close to the live edge. | +| `cache_duration` | `300` | `300` | Retention baseline, in seconds, for cache objects that are not protected by active use. | +| `cache_bytes` | `10GB` | `10GB` or larger | Global HLS cache budget. | +| `cache_bytes_per_session` | `512MB` | `512MB` | Per shared-session HLS cache budget. | +| `max_segments_prefetch` | `6` | `6` | Maximum prefetch queue depth per shared HLS session. | +| `max_concurrent_segment_fetches_per_session` | `2` | `2` | Maximum concurrent future segment fetches for one shared HLS session. | +| `max_concurrent_segment_fetches_global` | `64` | `64` | Global cap for concurrent future segment fetches across all HLS sessions. | +| `origin_manifest_timeout_ms` | `3000` | `3000` | Timeout for upstream HLS manifest fetches. | +| `origin_segment_timeout_ms` | `10000` | `10000` | Timeout for upstream HLS segment fetches. | +| `session_idle_timeout` | `300` | `300` | Access-lease validity window and idle cleanup baseline, in seconds. | +| `manifest_recovery_burst.level` | `off` | `off`, then `friendly` or `balanced` only if needed | Optional extra recovery pressure when manifest refresh needs help. | +| `segment_repair.max_level` | `off` | `off`, then `low` only if needed | Maximum codec-aware MPEG-TS repair level. | +| `segment_repair.apply_to_first_segments` | `1` | `1` | Number of visible TS objects checked per lease activation. Maximum is `6`. | +| `segment_repair.max_parallel_repairs` | `1` | `1` | Maximum concurrent repair jobs. Must not exceed `max_segments_prefetch` when repair is enabled. | +| `segment_repair.postprocess_timeout_ms` | `2000` | `2000` | Timeout for segment post-processing, including repair and watchdog work. Minimum is `100`. | +| `segment_repair.size_increase.low_percent` | `2` | `2` | Allowed size increase for low-level repair output. | +| `segment_repair.size_increase.medium_percent` | `5` | `5` | Allowed size increase for medium-level repair output. | +| `segment_repair.size_increase.high_percent` | `20` | `20` | Allowed size increase for high-level repair output. | +| `segment_repair.corrupt_segment_watchdog.mode` | `off` | `off` | Optional watchdog for residual TS packet-corrupt warnings: `off`, `detect_only`, `sanitize`, or `diagnostic`. | +| `segment_repair.corrupt_segment_watchdog.max_parallel_jobs` | `1` | `1` | Maximum watchdog jobs. Must not exceed `max_segments_prefetch` when watchdog mode is enabled. | + +Cache commits enforce both byte budgets. An individual object may not exceed either budget, and committed usage is +reconstructed from disk after restart. Origin manifests are limited to 2 MiB before parsing. + +Supported byte-size units for `cache_bytes` and `cache_bytes_per_session` are: + +- `B`; +- `KB`, `MB`, `GB`, `TB` as decimal units; +- `KiB`, `MiB`, `GiB`, `TiB` as binary units; +- no suffix, meaning bytes. + +## Choosing cache sizes + +Start with the defaults unless you already know your traffic pattern. + +A rough sizing approach: + +```text +required cache_bytes ≈ active_shared_channels × cache_bytes_per_session +``` + +Example: if 20 shared HLS channels can be active at the same time and each session may use up to `512MB`, a global budget +of `10GB` is the minimum starting point. Add extra room when segments are large, when `cache_duration` is high, or when +many users join at slightly different playback positions. + +Disk cache does not remove the need for network capacity. Increasing prefetch and concurrency can make playback smoother, +but it can also increase upstream and local I/O load. + +## Recommended profiles + +### Small or first-time setup + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" +``` + +Use the defaults first. They are conservative and easier to debug. + +### More concurrent viewers + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" + cache_bytes: "50GB" + cache_bytes_per_session: "1GB" + max_concurrent_segment_fetches_global: 128 +``` + +Use this only when the machine has enough disk throughput and upstream bandwidth. Keep the per-session fetch cap modest +unless profiling shows it is the bottleneck. + +### Unstable manifests + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" + manifest_recovery_burst: + level: "friendly" +``` + +Use recovery burst levels gradually. Start with `friendly`. Move to stronger levels only when logs show manifest refresh +instability and the provider can tolerate additional requests. + +### Problematic MPEG-TS segments + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" + segment_repair: + max_level: "low" + apply_to_first_segments: 1 + max_parallel_repairs: 1 +``` + +Keep segment repair disabled unless you have real evidence of broken TS segments. Repair costs CPU and can add latency. + +## Validation rules + +Tuliprox validates HLS cache settings during configuration preparation. + +| Rule | Reason | +| :--- | :--- | +| `cache_duration >= 1` | Avoids immediate cache expiry. | +| `max_concurrent_segment_fetches_per_session >= 1` | At least one per-session fetch worker must be possible. | +| `max_concurrent_segment_fetches_global >= 1` | At least one global fetch worker must be possible. | +| `origin_manifest_timeout_ms >= 1` | Timeout must be non-zero. | +| `origin_segment_timeout_ms >= 1` | Timeout must be non-zero. | +| `session_idle_timeout >= 1` | Access leases and sessions need a positive validity window. | +| `segment_repair.apply_to_first_segments <= 6` | Limits startup repair work. | +| `segment_repair.postprocess_timeout_ms >= 100` | Avoids unrealistic post-processing timeouts. | +| `segment_repair.size_increase.* <= 100` | Prevents unbounded repaired-object growth. | +| repair enabled: `max_parallel_repairs >= 1` | Repair needs at least one worker. | +| repair enabled: `max_parallel_repairs <= max_segments_prefetch` | Repair concurrency must not exceed the session fetch window. | +| watchdog enabled: `max_parallel_jobs <= max_segments_prefetch` | Watchdog concurrency must stay within the session fetch window. | + +## Interaction with older HLS settings + +`reverse_proxy.stream.hls_session_ttl_secs` belongs to the older HLS request continuity path. It keeps a virtual provider +slot open between legacy HLS segment requests. + +Shared HLS uses `reverse_proxy.hls_cache.session_idle_timeout` for access-lease validity and idle cleanup. Do not tune +`hls_session_ttl_secs` when the problem is Shared HLS lease expiry. + +## Hot reload and restarts + +Configuration reloads can update HLS runtime settings. Changing the HLS cache path is a disruptive change: Tuliprox must +clear in-memory HLS sessions, access leases, QoS state, and repair state that point at the old location. + +Changing `rewrite_secret` is also disruptive for public HLS identifiers. Existing player URLs may reference old +`proxy_session_id` or transient resource IDs and should be treated as stale. diff --git a/docs/src/configuration/shared-hls-runtime-flow.md b/docs/src/configuration/shared-hls-runtime-flow.md new file mode 100644 index 000000000..091ea5dff --- /dev/null +++ b/docs/src/configuration/shared-hls-runtime-flow.md @@ -0,0 +1,215 @@ +# Shared HLS Runtime Flow + +This page explains what happens after a player opens a live HLS URL on a target that has Shared HLS enabled. + +For the operator setup, see [Shared HLS Configuration](./shared-hls-configuration.md). For the compact state-machine +reference, see [HLS Cache State Machines](./hls-cache-state-machine.md). + +## Important identifiers + +| Identifier | Scope | Created by | Purpose | +| :--- | :--- | :--- | :--- | +| `HlsSessionKey` | Shared content | Built from `input_id`, the literal HLS kind, and `stream_ref`. | Stable internal key for one shared live HLS session. It does not include provider username, password, or origin URL. | +| `proxy_session_id` | Public shared content URL | Derived from `HlsSessionKey` and the configured secret. | Opaque URL token for the shared session. The same input and stream reference produce the same public session ID while the secret stays stable. | +| `HlsPlaybackFamilyKey` | Playback family | Built from Tuliprox username and client fingerprint. | Groups related playback attempts by user and client. | +| `hls_access_lease_id` | Per playback URL | Random server-side lease ID. | Lets exactly one playback path use the shared session after Tuliprox validates the user and lease. | +| HLS cache user session token | Per playback admission | Generated for the Shared HLS entry request. | Connects the access lease to Tuliprox user session accounting and connection handling. | + +The key design point is that `proxy_session_id` is shared, but `hls_access_lease_id` is not. + +## End-to-end request flow + +```mermaid +sequenceDiagram + participant Player + participant Entry as Generated HLS entry URL + participant Lease as Access lease store + participant Session as Shared HLS session + participant Origin as Upstream provider + participant Cache as HLS cache + + Player->>Entry: GET generated live HLS URL + Entry->>Entry: Authenticate user, resolve target/input/stream + Entry->>Lease: Create Pending HlsAccessLease + Entry-->>Player: 307 Location: /hls/shared/live/.../manifest.m3u8 + + Player->>Session: GET canonical manifest + Session->>Lease: Validate manifest access + Session->>Session: Create or reuse HlsSession + Session->>Origin: Fetch or refresh origin manifest if needed + Origin-->>Session: Origin manifest + Session->>Cache: Commit timeline, queue prefetch + Session-->>Player: Render manifest with this access lease + + Player->>Session: GET segment/MAP/resource + Session->>Lease: Activate or refresh access lease + Session->>Cache: Read cached object or schedule demand fetch + Cache->>Origin: Fetch missing object if needed + Session-->>Player: Media response +``` + +## Step 1: Entry request + +The player first opens a generated Tuliprox live HLS URL. Tuliprox resolves the target, input, virtual ID, stream URL, and +user connection permission. + +If Shared HLS is enabled for the target, Tuliprox creates a fresh `Pending` `HlsAccessLease` for this playback and returns +an HTTP `307 Temporary Redirect` to the canonical shared manifest path: + +```text +/hls/shared/live///manifest.m3u8 +``` + +The redirect response uses `Cache-Control: no-store`, because the lease ID is user-specific and should not be cached by a +browser, proxy, or IPTV client. + +## Step 2: Canonical manifest request + +The player follows the redirect and requests the canonical manifest. Tuliprox validates the access lease, restores the +user context, checks admission, then creates or reuses the shared `HlsSession` for the `proxy_session_id`. + +A manifest request can keep a lease alive, but it does not necessarily prove active playback. For that reason, a manifest +request for a `Pending` lease usually keeps it pending. Media requests are what activate the lease. + +The manifest response is rendered with the current `hls_access_lease_id` inserted into segment, MAP, and transient +resource URLs. This is how Tuliprox keeps the shared content session while preserving per-user access control. + +## Step 3: Segment, MAP, and resource requests + +The player then requests media URLs from the rendered manifest: + +```text +/hls/shared/live/// +/hls/shared/live///map/ +/hls/shared/live///r/ +``` + +For these requests, Tuliprox validates the lease again and activates or refreshes it. A valid resource request moves the +lease to `Activated`, extends its active window, and extends its validity window. + +Tuliprox serves the object from one of these paths: + +| Path | When used | +| :--- | :--- | +| Ready cache hit | The segment or MAP object is already committed and readable. | +| Demand fetch | The manifest references an object that is known but not ready yet. | +| Prefetch | Background work fetches likely next segments after a manifest commit. | +| Transient passthrough | The manifest contains a resource that cannot be handled as a normal cached timeline object. | + +Range requests are supported for ready cached objects. This matters for clients that seek or reconnect inside an HLS +segment. + +## Access lease lifecycle + +```mermaid +stateDiagram-v2 + [*] --> Pending: Entry redirect creates lease + Pending --> Pending: Manifest access + Pending --> Activated: Segment, MAP, or transient resource access + Pending --> Denied: User/provider admission denied + Pending --> Expired: Pending deadline reached + + Activated --> Activated: Media access or active manifest touch + Activated --> Idle: Active window elapsed + Activated --> Expired: Validity window elapsed + Activated --> Denied: Admission denied + + Idle --> Idle: Manifest touch extends validity + Idle --> Activated: Later media access + Idle --> Expired: Validity window elapsed + Idle --> Denied: Admission denied + + Expired --> [*]: Removed by lifecycle cleanup + Denied --> [*]: Removed by lifecycle cleanup +``` + +Important timing rules: + +| Rule | Meaning | +| :--- | :--- | +| Initial pending bootstrap window | A newly redirected lease can wait for the first useful manifest decision. The current bootstrap window is 90 seconds. | +| Pending follow-up window | After a pending manifest response, Tuliprox can shorten the pending window to `max(10 seconds, 2 * target_duration)`. | +| Active window | A media request keeps a lease active for `2 * target_duration`. If target duration is unknown, Tuliprox uses a 15-second fallback. | +| Validity window | The lease validity baseline comes from `hls_cache.session_idle_timeout`. Default: 300 seconds. | + +An `Idle` lease can still validate its own URL, but it does not keep origin prefetch work active until a later media +request reactivates it. + +## Shared session lifecycle + +`HlsSession` is a composite runtime object rather than a single `Active` enum. Its effective state comes from store +presence, media mode, access leases, origin work, cache readers, and the garbage-collection marker. + +```mermaid +stateDiagram-v2 + [*] --> Missing + Missing --> NormalCacheTimeline: First accepted canonical manifest + NormalCacheTimeline --> NormalCacheTimeline: Normal manifest commit and segment cache + NormalCacheTimeline --> TransientPassthrough: Manifest requires transient handling + TransientPassthrough --> TransientPassthrough: Transient manifest/resource handling + NormalCacheTimeline --> Removing: Idle cleanup conditions hold + TransientPassthrough --> Removing: Idle cleanup conditions hold + Removing --> Missing: Store entry and cache directory removed + Removing --> NormalCacheTimeline: Cleanup deferred by active work + Removing --> TransientPassthrough: Cleanup deferred by active work +``` + +A session can be removed only when all relevant work has drained: + +- no active access lease keeps it alive; +- no origin manifest refresh is active; +- no segment or MAP fetch is active; +- no prefetch queue entries are pending; +- no ready object has an active reader; +- no transient resource reader is active; +- no temporary files block cleanup. + +## Manifest commit and startup rendering + +Tuliprox uses the origin manifest to build a shared internal timeline. A rendered user manifest contains only the visible +window and user-specific access-lease URLs. + +Startup rendering can apply `hls_cache.strip` to hold back the initial view. This can help players that behave poorly when +they start too close to the live edge. + +Manifest acceptance is conservative. Tuliprox tracks media sequence progress, host changes, and recovery signals so it can +avoid accepting a manifest that would move the shared session backwards or suddenly switch to an unsafe origin host. + +## Origin account binding + +The shared session can hold an origin account binding. That binding represents the provider account or provider connection +Tuliprox uses for origin work. + +Origin account protection is based on successful media access: + +| Protection state | Meaning | +| :--- | :--- | +| `NoMediaYet` | No segment, MAP, or transient resource response has succeeded yet. | +| `HardActive` | Recent media access is still inside the hard-active window. | +| `SoftActive` | The hard window has elapsed, but the session is still inside the soft overlap window. | +| `Expired` | The soft window elapsed and the binding is no longer protected by recent media activity. | + +Upgrades to the effective origin policy apply immediately. Downgrades or clearing the policy wait for a target-duration +based grace window so active playback is not destabilized by a short pause or by brief client behavior differences. + +## Transient passthrough flow + +Some manifests contain objects that should not become normal cached timeline segments. Examples include certain key +resources or unsupported HLS features. + +When Tuliprox detects such a manifest, the session can switch from `NormalCacheTimeline` to `TransientPassthrough`. +The rendered manifest then points user-specific `/r/` URLs at short-lived transient resource entries. +Tuliprox still validates the `hls_access_lease_id` before serving those resources. + +## Cleanup and stale URLs + +A player may keep old HLS URLs after a pause, device sleep, app restart, or network change. When the corresponding access +lease or session is gone, the old URL no longer represents a valid server-side object. + +Typical outcomes are: + +- a redirect to the `hls_session_or_lease_expired` custom video; +- `404 Not Found` for a stale shared HLS path; +- `503 Service Unavailable` with `Retry-After` when origin/cache work is temporarily not ready. + +Players should reopen the original generated playlist entry URL when they recover from a long pause or stale HLS path. diff --git a/docs/src/configuration/shared-hls-sessions.md b/docs/src/configuration/shared-hls-sessions.md new file mode 100644 index 000000000..48a001f9f --- /dev/null +++ b/docs/src/configuration/shared-hls-sessions.md @@ -0,0 +1,204 @@ +# Shared HLS Sessions + +Shared HLS Sessions let several viewers watch the same live HLS channel through one shared Tuliprox HLS cache session. +This reduces duplicated upstream traffic and lowers the chance of hitting provider connection limits, while each viewer +still keeps their own Tuliprox user access lease. + +This page is the recommended starting point for operators who are new to the feature. For every available setting, see +[Shared HLS Configuration](./shared-hls-configuration.md). For internal state machines, see +[HLS Cache State Machines](./hls-cache-state-machine.md). + +## What problem does this solve? + +A live HLS stream is not one long connection. It is a playlist file, usually called a manifest, plus many small media +files, usually called segments. + +Without sharing, every viewer can cause Tuliprox to fetch the same manifest and the same segment files from the upstream +provider. With Shared HLS Sessions, Tuliprox creates one shared server-side HLS session for the same live channel and +serves the users from that shared session whenever possible. + +A simple analogy: + +| Concept | Everyday analogy | Meaning in Tuliprox | +| :--- | :--- | :--- | +| `HlsSession` | One cinema screen | The shared live HLS cache session for one channel. | +| `HlsAccessLease` | One ticket | The per-viewer permission to use that shared session. | +| Provider account binding | The cinema’s source feed | The upstream account/connection Tuliprox uses to fetch the live channel. | +| HLS cache | Local snack counter | Segment and MAP files Tuliprox can serve locally after fetching them once. | + +## When should I enable it? + +Enable Shared HLS Sessions when these statements are true: + +- the stream type is live HLS; +- multiple users may watch the same channel at the same time; +- the provider limits simultaneous connections or penalizes frequent account switching; +- clients can follow HTTP redirects and play normal HLS playlists; +- Tuliprox reverse proxy mode is used instead of direct provider URLs. + +Do not expect this feature to share VOD, series, catchup, or MPEG-TS streams. MPEG-TS stream sharing is controlled by the +separate `share_live_streams.mpeg_ts` option. + +## What is shared and what is still per user? + +| Shared between viewers | Still separate per viewer | +| :--- | :--- | +| The stable `HlsSession` for the same input and live stream reference. | The `hls_access_lease_id` in the user-facing URL. | +| Origin manifest refreshes and origin segment fetches when the cache can reuse them. | Tuliprox username, client fingerprint, and access checks. | +| Cached HLS segments and HLS MAP objects. | User connection admission and stream reservation lifecycle. | +| Transient HLS resource handling when the manifest requires it. | User-visible error handling and custom video redirects. | + +This means the feature is not a bypass for user limits. It is a way to avoid duplicated upstream HLS work after Tuliprox +has accepted each viewer. + +## Requirements + +Shared HLS starts only when both switches are enabled: + +1. `reverse_proxy.hls_cache` exists in `config.yml`. +2. The target has `options.share_live_streams.hls: true` in `source.yml`. + +A stable `reverse_proxy.rewrite_secret` is strongly recommended. Changing the secret changes future public HLS session +IDs and transient resource IDs, so existing player URLs may stop matching server-side runtime state. + +The cache directory must be writable by the Tuliprox process and must have enough free disk space for the configured HLS +cache budgets. + +## Minimal configuration + +`config.yml`: + +```yaml +reverse_proxy: + rewrite_secret: "00112233445566778899aabbccddeeff" + hls_cache: + cache_path: "/var/lib/tuliprox/cache/hls" + cache_bytes: "10GB" + cache_bytes_per_session: "512MB" + session_idle_timeout: 300 +``` + +`source.yml`: + +```yaml +targets: + - name: xc_m3u + output: + - type: xtream + - type: m3u + options: + share_live_streams: + hls: true + mpeg_ts: false +``` + +`share_live_streams.hls` and `share_live_streams.mpeg_ts` are independent. Use the object form shown above. The old style +`share_live_streams: true` is not valid for this configuration shape. + +## Request flow in plain language + +1. A player opens the normal generated Tuliprox HLS live URL. +2. Tuliprox authenticates the user and checks whether this target allows Shared HLS. +3. Tuliprox creates a new access lease for that playback. +4. Tuliprox returns an HTTP `307 Temporary Redirect` to the canonical shared HLS manifest URL. +5. The player follows the redirect and requests `/hls/shared/live/.../manifest.m3u8`. +6. Tuliprox creates or reuses the shared `HlsSession` for the live channel. +7. The player requests segments, MAP files, or transient resources through the same access lease. +8. Segment and MAP requests activate or refresh the lease. When the player stops, the lease idles and is later removed. +9. When no usable lease and no origin/cache work remain, the shared session is eligible for cleanup. + +```mermaid +flowchart TD + A["Generated HLS live URL"] --> B["User and target checks"] + B --> C{"Shared HLS enabled?"} + C -->|no| D["Legacy HLS reverse proxy path"] + C -->|yes| E["Create per-playback HLS access lease"] + E --> F["307 redirect to canonical shared manifest"] + F --> G["Create or reuse shared HLS session"] + G --> H["Render manifest with this user's access lease"] + H --> I["Serve segments, MAPs, or resources from cache or origin"] +``` + +Canonical Shared HLS URLs look like this: + +```text +/hls/shared/live///manifest.m3u8 +/hls/shared/live/// +/hls/shared/live///map/ +/hls/shared/live///r/ +``` + +The `proxy_session_id` identifies the shared content session. The `hls_access_lease_id` identifies one viewer’s +server-side access lease. + +## What happens when the player pauses or stops? + +A lease can be `Pending`, `Activated`, `Idle`, `Expired`, or `Denied`. + +Manifest requests alone do not necessarily mean that a user is actively watching media. Segment, MAP, and transient +resource requests are the stronger signal that playback is active. Once a lease no longer receives active media requests, +it can move to `Idle`. After the configured validity window, it becomes `Expired` and is removed. + +The shared `HlsSession` survives as long as it still has usable leases or active origin/cache work. This avoids deleting a +session while another user is still watching or while Tuliprox is still finishing cache work. + +## How provider connections are protected + +Shared HLS cooperates with Tuliprox connection handling: + +- user admission is still checked per viewer; +- the shared origin session can reuse a provider account binding for the same HLS session owner; +- soft or low-priority connections can still be denied or preempted according to the connection policy; +- hard-active sessions are protected from speculative account reuse; +- soft-active sessions may be candidates for controlled overlap only when the runtime determines it is safe. + +For the broader connection-handling model, see [Connection Handling](./connection-handling.md). + +## Cache behavior + +Tuliprox stores HLS segment and MAP cache objects below `reverse_proxy.hls_cache.cache_path`. The cache is bounded by: + +- `cache_bytes`, the global HLS cache budget; +- `cache_bytes_per_session`, the per shared-session budget; +- `cache_duration`, the retention baseline for unprotected objects; +- active reader and origin-work protection, so files are not removed while in use. + +The HLS cache is separate from `reverse_proxy.cache`, which is used for logos, images, and other rewritten resources. + +## Transient passthrough mode + +Some HLS manifests contain resources that cannot safely be represented as normal cached timeline segments. Examples are +certain encryption-key resources or unsupported HLS tags. In those cases, Tuliprox can switch the session into transient +passthrough mode. + +In transient mode, Tuliprox still validates the user’s access lease. It then fetches and serves the needed resource +through a short-lived, controlled resource path instead of pretending that the resource is a normal cached segment. + +## User-visible fallback behavior + +When a Shared HLS request can no longer be served, Tuliprox either returns a direct HTTP error or redirects to a configured +custom video, depending on the failure phase. + +Common custom-video cases include: + +| Situation | Typical user-facing result | +| :--- | :--- | +| User connection limit exhausted | `user_connections_exhausted` custom video. | +| Provider account limit exhausted | `provider_connections_exhausted` custom video. | +| Low-priority stream preempted | `low_priority_preempted` custom video. | +| Shared HLS session or lease expired | `hls_session_or_lease_expired` custom video. | +| Origin manifest or media becomes unavailable | `channel_unavailable` custom video. | + +For practical debugging steps, see [Shared HLS Troubleshooting](./shared-hls-troubleshooting.md). + +## Operator checklist + +Before enabling Shared HLS for a target: + +- set a stable 32-character hex `reverse_proxy.rewrite_secret`; +- add `reverse_proxy.hls_cache` to `config.yml`; +- use a cache path that survives restarts if you want fewer cold starts; +- make sure the Tuliprox process can create and delete files in the cache path; +- enable `options.share_live_streams.hls: true` only on targets that should use Shared HLS; +- start with the default HLS cache limits, then tune only after observing real traffic; +- watch logs for `HLS session created`, `HLS session reused`, and `HLS access lease rejected`. diff --git a/docs/src/configuration/shared-hls-troubleshooting.md b/docs/src/configuration/shared-hls-troubleshooting.md new file mode 100644 index 000000000..b3ae7bd65 --- /dev/null +++ b/docs/src/configuration/shared-hls-troubleshooting.md @@ -0,0 +1,226 @@ +# Shared HLS Troubleshooting + +This page helps operators diagnose Shared HLS problems without reading the source code. + +Start with the checklist, then use the symptom sections below. + +## Quick checklist + +1. Confirm `reverse_proxy.hls_cache` exists in `config.yml`. +2. Confirm the target has `options.share_live_streams.hls: true` in `source.yml`. +3. Confirm the HLS cache path exists or can be created by the Tuliprox process. +4. Confirm `reverse_proxy.rewrite_secret` is stable and has not changed between restarts. +5. Confirm the client follows HTTP `307 Temporary Redirect` responses. +6. Confirm the client is opening the generated Tuliprox HLS URL, not a stale `/hls/shared/live/...` URL from a previous playback. +7. Check logs for `HLS access lease rejected`, `HLS session created`, `HLS session reused`, and `HLS access leases marked channel unavailable`. + +## How to recognize Shared HLS in URLs + +Shared HLS URLs contain `/hls/shared/live/`: + +```text +/hls/shared/live///manifest.m3u8 +``` + +The first token identifies the shared session. The second token is the per-playback access lease. + +A healthy first request usually looks like this: + +```text +GET /hls/...original generated live URL... +307 Temporary Redirect +Location: /hls/shared/live///manifest.m3u8 +``` + +## Symptom: the player never reaches `/hls/shared/live/` + +Likely causes: + +| Cause | Check | Fix | +| :--- | :--- | :--- | +| Global cache block missing | Search `config.yml` for `reverse_proxy.hls_cache`. | Add the `hls_cache` block. | +| Target switch missing | Search the target in `source.yml`. | Add `options.share_live_streams.hls: true`. | +| Client uses direct source URL | Look at the URL in the player or access log. | Use the generated Tuliprox playlist URL. | +| Client does not follow redirects | Look for the initial `307` without a follow-up request. | Use a client that supports redirects, or fix reverse proxy/client behavior. | + +## Symptom: `404 Not Found` on `/hls/shared/live/...` + +A `404` usually means the URL does not match live server-side state. + +Common reasons: + +- the player reused an old manifest, segment, MAP, or resource URL after the lease expired; +- Tuliprox restarted and in-memory leases were lost; +- the HLS cache path changed and Tuliprox cleared runtime HLS state; +- `rewrite_secret` changed and future public session IDs no longer match old URLs; +- the URL was copied from one user or client to another. + +Fix: + +1. Make the player reopen the original generated Tuliprox playlist entry URL. +2. Check whether Tuliprox restarted or reloaded a changed HLS cache path. +3. Keep `rewrite_secret` stable. +4. Do not share canonical `/hls/shared/live/...` URLs between users. + +## Symptom: custom video `hls_session_or_lease_expired` + +This means the player requested a Shared HLS path whose access lease or shared session is no longer usable. + +Common causes: + +- long pause or device sleep; +- IPTV app resumed with old segment URLs instead of reopening the playlist; +- `session_idle_timeout` is too low for the client’s pause/reconnect behavior; +- Tuliprox was restarted while the player kept old URLs. + +Fix: + +- Ask the client to reload the channel from the playlist. +- Keep `session_idle_timeout` at the default `300` seconds unless you have a reason to lower it. +- Increase `session_idle_timeout` only if stale-resume behavior is common and you can afford longer runtime retention. + +## Symptom: custom video `channel_unavailable` + +This means Tuliprox accepted the user request, but the shared HLS session could not currently produce a usable channel +response. + +Possible causes: + +| Area | What to look for | +| :--- | :--- | +| Manifest fetch | `HLS manifest temporary failure counted` or `HLS manifest temporary failure threshold reached`. | +| Manifest commit | `HLS access lease marked channel unavailable after fresh manifest commit failed`. | +| Segment fetch | `HLS segment temporary failure counted`, `HLS segment temporary failure threshold reached`, or permanent segment failures. | +| MAP/transient resource fetch | MAP or transient object failure messages. | +| Provider account | Origin account unavailable or provider lineup exhaustion. | + +Fix: + +1. Test the origin HLS URL from the Tuliprox host. +2. Check provider account limits and provider availability. +3. If the provider produces unstable manifests, try `manifest_recovery_burst.level: "friendly"` before using stronger levels. +4. If TS segments are corrupt, try `segment_repair.max_level: "low"` only after confirming segment errors in logs. + +## Symptom: custom video `user_connections_exhausted` + +Tuliprox rejected the playback because the Tuliprox user has no available stream capacity. + +Shared HLS does not bypass user limits. Each viewer still needs an accepted Tuliprox user session and access lease. + +Fix: + +- Check the user’s configured connection limit. +- Check whether old clients are keeping sessions alive. +- Review active streams in the Tuliprox UI/API if available. + +## Symptom: custom video `provider_connections_exhausted` + +Tuliprox could not acquire a provider account or provider connection for origin work. + +Shared HLS reduces duplicate origin work after a session exists, but the first session for a channel still needs origin +access. + +Fix: + +- Check provider account connection limits. +- Check whether other live, catchup, or non-HLS sessions are using the same provider accounts. +- Review priority and preemption settings. +- Confirm that the target really shares HLS. If not, multiple viewers may be opening separate legacy HLS paths. + +## Symptom: low-priority stream is interrupted + +Low-priority or soft connections can be preempted according to the normal connection-handling policy. + +Fix: + +- Review user and target priority settings. +- See [Priorities, Soft Connections and Preemption](./connection-handling-priorities-and-preemption.md). +- Check for `low_priority_preempted` custom-video redirects. + +## Symptom: repeated `503 Service Unavailable` with `Retry-After` + +A `503` with `Retry-After` can be normal during short origin/cache waits, especially near startup or when a requested +segment is known but not ready yet. + +Investigate when it repeats for too long: + +| Log signal | Interpretation | +| :--- | :--- | +| `HLS segment demand fetch skipped by backpressure` | The session or global fetch limit is saturated. | +| `HLS segment temporary failure counted` | The origin segment fetch is failing temporarily. | +| `HLS manifest marked fresh-commit required after hard fetch failure` | Manifest refresh had a hard failure and needs a fresh commit. | +| `HLS access leases marked channel unavailable` | Tuliprox gave up for the current lease/session after repeated failures. | + +Fix: + +- Keep default concurrency until the real bottleneck is known. +- Increase `max_concurrent_segment_fetches_global` only when CPU, disk, and provider bandwidth can handle it. +- Avoid setting per-session concurrency too high; one problematic channel can otherwise dominate origin work. + +## Symptom: cache directory grows too large + +Check these settings: + +| Setting | Effect | +| :--- | :--- | +| `cache_bytes` | Global HLS cache budget. | +| `cache_bytes_per_session` | Per-session budget. | +| `cache_duration` | Retention baseline for unprotected objects. | +| `session_idle_timeout` | Longer timeouts keep sessions and lease-related state around longer. | + +Fix: + +- Lower `cache_bytes_per_session` if single channels grow too large. +- Lower `cache_duration` if unprotected old objects stay too long. +- Put `cache_path` on a disk with enough free space and predictable cleanup behavior. + +## Symptom: cache path permission errors + +The Tuliprox process must be able to create directories, write temporary files, rename committed cache objects, read cached +objects, and delete expired files. + +Fix: + +```bash +mkdir -p /var/lib/tuliprox/cache/hls +chown -R tuliprox:tuliprox /var/lib/tuliprox/cache/hls +chmod 750 /var/lib/tuliprox/cache/hls +``` + +Adapt the user and group to your deployment. + +## Useful log messages + +| Message fragment | Meaning | +| :--- | :--- | +| `HLS access lease prepared` | Entry request created a new access lease and returned the shared HLS redirect. | +| `HLS access lease accepted` | Manifest or media request validated the lease. | +| `HLS access lease rejected` | Lease validation or admission failed. Check the reason suffix. | +| `HLS access lease idled` | Lease left active media state and released its active stream reservation. | +| `HLS access lease removed` | Lifecycle cleanup removed the lease. | +| `HLS session created` | A new shared session was created for a session key. | +| `HLS session reused` | Existing shared session was reused for the same session key. | +| `HLS session lifecycle expired` | Idle shared session was cleaned up. | +| `HLS lifecycle state snapshot` | Debug summary of sessions, leases, QoS, repair, and cleanup state. | +| `HLS manifest rendered` | A shared manifest was successfully rendered for a player. | +| `HLS segment cached` | A segment was fetched and committed to the cache. | +| `HLS session switched to transient passthrough` | The manifest required transient resource handling. | + +## Safe first debugging command sequence + +Use equivalent commands for your deployment: + +```bash +# 1. Confirm config switches +grep -R "hls_cache" config.yml +grep -R "share_live_streams" source.yml + +# 2. Confirm cache path permissions +ls -ld /var/lib/tuliprox/cache/hls + +# 3. Watch HLS-related logs +grep -i "HLS " tuliprox.log | tail -200 +``` + +If the issue is client-specific, compare a working player and a failing player. Many HLS issues are caused by stale URL +reuse, missing redirect support, aggressive caching, or reconnect behavior in the client. diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index eb4d7b474..0eda045e7 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -661,7 +661,9 @@ sources: sort: { } options: ignore_logo: false - share_live_streams: false + share_live_streams: + hls: false + mpeg_ts: false remove_duplicates: false output: - type: xtream @@ -730,7 +732,7 @@ Valid values are: The target-level `filter` is a string-based expression using Tuliprox's filter DSL. It defines which entries remain in the final target after the selected processing stages have been applied. -You can define complex strings or regex patterns exactly once in [template.yml](./configuration/template.md) +You can define complex strings or regex patterns exactly once in [template.yml](./template.md) and call them by wrapping the template name in exclamation marks: `!MACRO_NAME!`. For less verbose expression definitions, inline filter definitions are also supported. @@ -930,20 +932,30 @@ targets: use_output: xtream options: ignore_logo: false - share_live_streams: true + share_live_streams: + hls: true + mpeg_ts: true remove_duplicates: false ``` #### Target Option Parameters -| Parameter | Type | Required | Default | Technical Impact & Background | -|:---------------------|:-----|:--------:|:--------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ignore_logo` | Bool | No | `false` | Ignores `tvg-logo` and `tvg-logo-small` attributes. This reduces downstream device-side logo caching and can keep generated M3U playlists leaner for clients with limited storage or poor cache invalidation behavior. | -| `share_live_streams` | Bool | No | `false` | Allows Tuliprox to share live stream connections in reverse proxy mode. This can reduce upstream provider connection usage when multiple clients watch the same channel, but it increases memory usage per shared channel. | -| `remove_duplicates` | Bool | No | `false` | Attempts to remove duplicate entries by `url`. This improves playlist cleanliness and reduces confusing duplicates in the client-facing output. | -| `force_redirect` | Bool | No | `false` | Optional redirect-related behavior switch. This influences how Tuliprox serves final stream delivery where redirect-style output handling is required by the deployment model. | +| Parameter | Type | Required | Default | Technical Impact & Background | +|:-----------------------------|:-----|:--------:|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ignore_logo` | Bool | No | `false` | Ignores `tvg-logo` and `tvg-logo-small` attributes. This reduces downstream device-side logo caching and can keep generated M3U playlists leaner for clients with limited storage or poor cache invalidation behavior. | +| `share_live_streams.hls` | Bool | No | `false` | Enables HLS live sharing for the new HLS cache proxy path. This is a configuration switch for the HLS cache feature and is independent from MPEG-TS stream sharing. | +| `share_live_streams.mpeg_ts` | Bool | No | `false` | Allows Tuliprox to share MPEG-TS live stream connections in reverse proxy mode. This can reduce upstream provider connection usage when multiple clients watch the same channel, but it increases memory usage per shared channel. | +| `remove_duplicates` | Bool | No | `false` | Attempts to remove duplicate entries by `url`. This improves playlist cleanliness and reduces confusing duplicates in the client-facing output. | +| `force_redirect` | Bool | No | `false` | Optional redirect-related behavior switch. This influences how Tuliprox serves final stream delivery where redirect-style output handling is required by the deployment model. | -> **⚠️ Warning:** When `share_live_streams` is enabled, each shared channel consumes at least **12 MB** of memory, +> **Shared HLS:** `share_live_streams.hls` requires `reverse_proxy.hls_cache` in `config.yml`. +> Start with [Shared HLS Sessions](./shared-hls-sessions.md) for the feature overview and +> [Shared HLS Configuration](./shared-hls-configuration.md) for the full setup checklist. +> +> Use the object form shown above. The old boolean style `share_live_streams: true` is not valid for this configuration, +> because HLS sharing and MPEG-TS sharing are independent switches. +> +> **⚠️ Warning:** When `share_live_streams.mpeg_ts` is enabled, each shared channel consumes at least **12 MB** of memory, > regardless of the number of connected clients. > If the reverse-proxy buffer size is increased above `1024`, memory usage increases accordingly. > Example: with a buffer size of `2048`, each shared channel consumes at least **24 MB**. diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index b792971bb..2b42c69c9 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -275,6 +275,29 @@ "ENABLED": "Globally enables or disables disk caching for input resources.", "SIZE": "Maximum size (in bytes) allowed for the cache directory. Allowed units are: B, KB, MB, GB, TB" }, + "HLS_CACHE_CONFIG": { + "CACHE_BYTES": "Total disk limit for HLS cache files.", + "CACHE_BYTES_PER_SESSION": "Disk limit per shared HLS session.", + "CACHE_DURATION": "Retention time for unprotected HLS cache files.", + "CACHE_PATH": "Base directory for HLS segment and MAP cache files.", + "MAX_CONCURRENT_SEGMENT_FETCHES_GLOBAL": "Global limit for parallel HLS segment fetches.", + "MAX_CONCURRENT_SEGMENT_FETCHES_PER_SESSION": "Per-session limit for parallel segment fetches.", + "MAX_SEGMENTS_PREFETCH": "Known origin segments to prefetch per session.", + "MANIFEST_RECOVERY_BURST": "Runs a small staggered burst on the first timeline-recovery attempt after a rejected manifest.", + "ORIGIN_MANIFEST_TIMEOUT_MS": "Timeout for origin manifest requests.", + "ORIGIN_SEGMENT_TIMEOUT_MS": "Timeout for origin segment requests.", + "SEGMENT_REPAIR_APPLY_TO_FIRST_SEGMENTS": "Segments checked per access lease.", + "SEGMENT_REPAIR_MAX_PARALLEL_REPAIRS": "Concurrent repair jobs. Must not exceed prefetch.", + "SEGMENT_REPAIR_MAX_LEVEL": "Maximum TS repair level allowed for trigger-derived repairs.", + "SEGMENT_REPAIR_POSTPROCESS_TIMEOUT_MS": "Timeout for the complete segment post-processing chain.", + "SEGMENT_REPAIR_SIZE_INCREASE": "Allowed segment size growth for the selected repair level.", + "SEGMENT_REPAIR_TRIGGER": "Automatic codec trigger policy selects the required repair level.", + "CORRUPT_SEGMENT_WATCHDOG": "Handles residual TS packet-corrupt warnings after regular repair policy.", + "CORRUPT_SEGMENT_WATCHDOG_MAX_PARALLEL_JOBS": "Concurrent watchdog sanitize jobs.", + "SESSION_IDLE_TIMEOUT": "Idle time before unused HLS sessions are removed.", + "STRIP_MODE": "How the initial playlist lags behind live edge.", + "STRIP_VALUE": "Gap value in segments or seconds." + }, "CONFIG_API": { "HOST": "Host where the tuliprox API listens. Default: 0.0.0.0", "PORT": "TCP port where the tuliprox API listens. Default: 8901", @@ -331,6 +354,13 @@ "USE_MEMORY_CACHE": "If enabled, the playlist is cached in RAM to reduce disk overhead (increases memory usage).", "WATCH": "List of group patterns to monitor for changes. Notifications are sent via configured messaging." }, + "CONFIG_TARGET_OPTIONS": { + "SHARE_LIVE_STREAMS": "Enables live stream sharing. The master toggle sets HLS and MPEG-TS together; each format can also be changed separately." + }, + "CONFIG_TARGET_SHARE_LIVE_STREAMS": { + "HLS": "Enables HLS live sharing for the new HLS cache proxy path.", + "MPEG_TS": "Enables MPEG-TS live sharing for the existing stream-sharing path." + }, "CONTENT_SECURITY_POLICY_CONFIG": { "CUSTOM_ATTRIBUTES": "Additional CSP attributes appended to the standard policy.", "ENABLED": "Enables or disables Content Security Policy headers for the WebUI." @@ -848,6 +878,7 @@ "ALL": "All", "ALLOWED_COUNTRIES": "Allowed Countries", "ALLOWED_NETWORKS": "Allowed Networks", + "APPLY_TO_FIRST_SEGMENTS": "Apply to first segments", "API": "Api", "API_CONFIG": "API", "API_CONFIGURATION": "API Configuration", @@ -872,9 +903,12 @@ "BUILD_TIME": "Build time", "BURST_SIZE": "Burst Size", "CACHE": "Cache", + "CACHE_BYTES": "Cache bytes", + "CACHE_BYTES_PER_SESSION": "Cache bytes per session", "CACHE_DIR": "Cache dir", "CACHE_DURATION": "Cache duration", "CACHE_DURATION_DAYS": "Cache Duration (days)", + "CACHE_PATH": "Cache path", "CANCEL": "Cancel", "CF_HEADER": "Remove Cloudflare headers", "CHANNEL": "Channel", @@ -1028,6 +1062,11 @@ "HEADER_NAME": "Header Name", "HEADER_VALUE": "Header Value", "HELP": "Help", + "HLS": "HLS", + "HLS_CACHE_PROXY": "HLS Cache Proxy", + "H264": "H.264", + "HEVC": "HEVC", + "HLS_CACHE_SEGMENT_REPAIR": "HLS Cache Segment Repair", "HOST": "Host", "ID": "Id", "IGNORE_LOGO": "Ignore logo", @@ -1079,6 +1118,10 @@ "MAX_BACKGROUND_PER_PROVIDER": "Max background per provider", "MAX_CON": "Max Con.", "MAX_CONNECTIONS": "Max Connections", + "MAX_CONCURRENT_SEGMENT_FETCHES_GLOBAL": "Max concurrent segment fetches global", + "MAX_CONCURRENT_SEGMENT_FETCHES_PER_SESSION": "Max concurrent segment fetches per session", + "MAX_SEGMENTS_PREFETCH": "Max segments prefetch", + "MAX_PARALLEL_REPAIRS": "Max concurrent repairs", "SOFT_PRIORITY": "Soft Priority", "SOFT_CONNECTIONS": "Soft Connections", "SOFT_CON": "Soft Con.", @@ -1119,6 +1162,7 @@ "MOVIE": "Movie", "MOVIE_CATEGORY": "Movie Category", "MSG_KIND_DISK_ALERT": "Disk Alert", + "MPEG_TS": "MPEG-TS", "MSG_KIND_ERROR": "Error", "MSG_KIND_INFO": "Info", "MSG_KIND_STATS": "Stats", @@ -1143,6 +1187,9 @@ "OPTIONS": "Options", "ORDER": "Order", "ORGANIZE_INTO_DIRECTORIES": "Organize Into Directories", + "ORIGIN_MANIFEST_TIMEOUT_MS": "Origin manifest timeout (ms)", + "MANIFEST_RECOVERY_BURST": "Manifest recovery burst", + "ORIGIN_SEGMENT_TIMEOUT_MS": "Origin segment timeout (ms)", "OUTPUT": "Output", "PANEL": "Panel", "PANEL_ACCOUNT_INFO": "Account info", @@ -1242,6 +1289,7 @@ "RESOLVE_DELAY_SEC": "Resolve Delay (sec)", "RESOLVE_FILTER": "Resolve Filter", "RESOLVE_TMDB": "Resolve TMDB", + "RESOURCE_IMAGE_CACHE": "Resource/Image Cache", "RESOURCE_RETRY": "Resource Retry", "RESOURCE_REWRITE_DISABLE": "Resource Rewrite disable", "RESOURCE_REWRITE_DISABLED": "Resource Rewrite Disabled", @@ -1252,6 +1300,11 @@ "REVERSE_PROXY": "Reverse Proxy", "REVERSE_PROXY_CONFIG": "Reverse Proxy", "REWRITE_SECRET": "Rewrite Secret", + "REPAIR_POLICY": "Policy", + "REPAIR_TRIGGER": "Repair trigger", + "POSTPROCESS_TIMEOUT_MS": "Postprocess timeout (ms)", + "CORRUPT_SEGMENT_WATCHDOG": "Corrupt segment watchdog", + "MAX_WATCHDOG_JOBS": "Max watchdog jobs", "SANITIZE_SENSITIVE_INFO": "Sanitize sensitive info", "SAVE": "Save", "SCAN_DIRECTORIES": "Scan Directories", @@ -1270,12 +1323,32 @@ "SERIES_SHORT": "S", "SERVER": "Server", "SERVER_TIME": "Server time", + "SEGMENT_REPAIR": "Max repair level", + "SEGMENT_REPAIR_AV_DESYNC": "A/V desync", + "SEGMENT_REPAIR_CODEC_PARAMETERS_MISSING": "Codec parameters missing", + "SEGMENT_REPAIR_DECODE_SLICE_HEADER_ERROR": "Decode slice header error", + "SEGMENT_REPAIR_INVALID_DATA": "Invalid data", + "SEGMENT_REPAIR_INVALID_METADATA_NALU": "Invalid metadata NALU", + "SEGMENT_REPAIR_INVALID_NAL": "Invalid NAL", + "SEGMENT_REPAIR_INVALID_VCL_NALU": "Invalid VCL NALU", + "SEGMENT_REPAIR_MISSING_PICTURE": "Missing picture", + "SEGMENT_REPAIR_MISSING_PPS": "Missing PPS", + "SEGMENT_REPAIR_MISSING_SPS": "Missing SPS", + "SEGMENT_REPAIR_MISSING_VPS": "Missing VPS", + "SEGMENT_REPAIR_MMCO_UNREF_SHORT_FAILURE": "MMCO unref short failure", + "SEGMENT_REPAIR_MULTIPLE_DOLBY_VISION_RPUS": "Multiple Dolby Vision RPUs", + "SEGMENT_REPAIR_NO_FRAME": "No frame", + "SEGMENT_REPAIR_PPS_ID_OUT_OF_RANGE": "PPS ID out of range", + "SEGMENT_REPAIR_REORDER_BUFFER_INCREASE": "Reorder buffer increase", + "SEGMENT_SIZE_INCREASE": "Segment size increase", "SETTINGS": "Settings", + "SESSION_IDLE_TIMEOUT": "Session idle timeout (secs)", "SHARED": "Shared", "SHARED_BURST_BUFFER_BYTES": "Shared burst buffer (MB)", "SHARE_LIVE_STREAMS": "Share live", "SHOW_SELECTED": "Show selected", "SIZE": "Size", + "SIZE_INCREASE": "Size increase", "SKIP": "Skip", "SKIP_DIRECT_SOURCE": "Skip direct source", "SKIP_LIVE": "Skip Live", @@ -1346,6 +1419,8 @@ "STREAM_HISTORY_WINDOW": "Window", "STREAM_ID": "Stream Id", "STRIP": "Strip", + "STRIP_MODE": "Strip mode", + "STRIP_VALUE": "Strip value", "STRM": "strm", "STRM_PROPS": "Properties", "SUBMIT": "Submit", @@ -1354,6 +1429,7 @@ "TARGETS": "Targets", "TECH": "Tech", "TELEGRAM": "Telegram", + "THRESHOLD_SCORE": "Threshold score", "TEMPLATES": "Templates", "TEMPLATE_PATH": "Template Path", "THROTTLE": "Throttle", diff --git a/frontend/scss/app/components/config/_config_view.scss b/frontend/scss/app/components/config/_config_view.scss index 61d711e09..cc268395c 100644 --- a/frontend/scss/app/components/config/_config_view.scss +++ b/frontend/scss/app/components/config/_config_view.scss @@ -137,6 +137,77 @@ } } +.tp__hls-cache-segment-repair { + &__box { + > .tp__titled-card { + padding-top: 0.6rem; + + .tp__titled-card__content > .tp__form-field:first-child { + margin-top: 0; + } + } + } +} + +.tp__hls-repair-slider { + display: grid; + grid-template-columns: minmax(0, 1fr) 2.35rem; + align-items: center; + gap: var(--gap-small); + width: 100%; + + &__range { + --tp-hls-repair-slider-fill: 0%; + + width: 100%; + height: 0.55rem; + margin: 0; + border-radius: var(--border-radius); + appearance: none; + -webkit-appearance: none; + background: linear-gradient( + to right, + var(--text-button-primary-background-color) 0%, + var(--text-button-primary-background-color) var(--tp-hls-repair-slider-fill), + var(--card-border-color) var(--tp-hls-repair-slider-fill), + var(--card-border-color) 100% + ); + cursor: pointer; + } + + &__range::-webkit-slider-thumb { + width: 1.15rem; + height: 1.15rem; + border: 1px solid var(--card-border-color); + border-radius: 50%; + appearance: none; + -webkit-appearance: none; + background-color: var(--text-button-primary-color); + box-shadow: var(--toggle-switch-toggle-shadow); + } + + &__range::-moz-range-thumb { + width: 1.15rem; + height: 1.15rem; + border: 1px solid var(--card-border-color); + border-radius: 50%; + background-color: var(--text-button-primary-color); + box-shadow: var(--toggle-switch-toggle-shadow); + } + + &__range::-moz-range-track { + height: 0.55rem; + border-radius: var(--border-radius); + background: transparent; + } + + &__value { + box-sizing: border-box; + min-width: 2.35rem; + text-align: center; + } +} + .tp__schedules-config-view { &__body { display: flex; diff --git a/frontend/scss/app/components/source_editor/_source_editor_form.scss b/frontend/scss/app/components/source_editor/_source_editor_form.scss index 5fbc2ce28..28eac6138 100644 --- a/frontend/scss/app/components/source_editor/_source_editor_form.scss +++ b/frontend/scss/app/components/source_editor/_source_editor_form.scss @@ -130,6 +130,40 @@ $form-width: 520px; margin-top: var(--margin-default); } + .tp__target-options { + display: flex; + flex-flow: column; + gap: var(--gap-default); + + &__group { + display: flex; + flex-flow: column; + gap: var(--gap-small); + } + + &__children { + display: flex; + flex-flow: column; + gap: var(--gap-default); + padding-left: var(--padding-default); + + .tp__form-field { + min-width: 0; + } + } + + &__heading { + display: flex; + align-items: center; + min-height: 2.5rem; + } + + &__child-content { + display: flex; + min-width: 0; + } + } + &__sidebar { .tp__icon-button.active { background-color: var(--menu-item-hover-background-color); diff --git a/frontend/src/app/components/config/config_update.rs b/frontend/src/app/components/config/config_update.rs index cee2ca98c..4bc0c2ef8 100644 --- a/frontend/src/app/components/config/config_update.rs +++ b/frontend/src/app/components/config/config_update.rs @@ -1,10 +1,10 @@ use crate::app::components::config::config_page::ConfigForm; use shared::{ + defaults::is_default_supported_library_extensions, model::{ ConfigDto, HdHomeRunConfigDto, LibraryConfigDto, LibraryMetadataConfigDto, LibraryPlaylistConfigDto, ThumbnailConfigDto, WebUiConfigDto, }, - utils::is_default_supported_library_extensions, }; macro_rules! set_config_field { diff --git a/frontend/src/app/components/config/macros.rs b/frontend/src/app/components/config/macros.rs index c40ca01ff..f73c1f9df 100644 --- a/frontend/src/app/components/config/macros.rs +++ b/frontend/src/app/components/config/macros.rs @@ -244,6 +244,38 @@ $crate::edit_field_text_option!(@inner $instance, $label, $field, $action, $hidd }}; } +#[macro_export] +macro_rules! edit_field_byte_size_option { + ($instance:expr, $label:expr, $field:ident, $action:path) => { +$crate::edit_field_byte_size_option!(@inner $instance, $label, $field, $action, false) + }; + ($instance:expr, $label:expr, $field:ident, $action:path, $hidden:expr) => { +$crate::edit_field_byte_size_option!(@inner $instance, $label, $field, $action, $hidden) + }; + (@inner $instance:expr, $label:expr, $field:ident, $action:path, $hidden:expr) => {{ + let instance = $instance.clone(); + html! { +
+ <$crate::app::components::input::Input + label={$label} + hidden={$hidden} + name={stringify!($field)} + field_id={Some($crate::app::components::dto_field_id(&instance.form, stringify!($field)))} + autocomplete={true} + value={instance.form.$field.as_ref().map_or_else(String::new, |v|v.to_string())} + on_change={Callback::from(move |value: String| { + instance.dispatch($action(if value.is_empty() { + None + } else { + Some(shared::model::ByteSize::new(value)) + })); + })} + /> +
+ } + }}; +} + #[macro_export] macro_rules! edit_field_textarea_option { ($instance:expr, $label:expr, $field:ident, $action:path) => {{ diff --git a/frontend/src/app/components/config/reverse_proxy_config_view.rs b/frontend/src/app/components/config/reverse_proxy_config_view.rs index 3d73bf68a..166332190 100644 --- a/frontend/src/app/components/config/reverse_proxy_config_view.rs +++ b/frontend/src/app/components/config/reverse_proxy_config_view.rs @@ -12,28 +12,36 @@ use crate::{ parse_admission_strategy_tags, remove_admission_strategy_tag, use_emit_mapped_option, AdmissionStrategiesDto, }, - Card, Chip, IconButton, RadioButtonGroup, TextButton, + dto_field_id, + number_input::NumberInput, + Card, Chip, DropDownOption, DropDownSelection, IconButton, RadioButtonGroup, Select, TextButton, }, context::ConfigContext, }, config_field, config_field_bool, config_field_child, config_field_custom, config_field_hide, config_field_optional, - edit_field_bool, edit_field_list, edit_field_number, edit_field_number_f64, edit_field_number_u16, - edit_field_number_u64, edit_field_number_usize, edit_field_text, edit_field_text_option, generate_form_reducer, + edit_field_bool, edit_field_byte_size_option, edit_field_list, edit_field_number, edit_field_number_f64, + edit_field_number_u16, edit_field_number_u64, edit_field_number_usize, edit_field_text, edit_field_text_option, + generate_form_reducer, i18n::{use_translation, YewI18n}, }; use shared::{ + defaults::default_secret, model::{ - CacheConfigDto, GeoIpConfigDto, GeoIpUnavailablePolicy, QosAggregationConfigDto, RateLimitConfigDto, - ResourceRetryConfigDto, ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, StreamBufferConfigDto, - StreamConfigDto, StreamHistoryConfigDto, + ByteSize, CacheConfigDto, GeoIpConfigDto, GeoIpUnavailablePolicy, HlsCacheConfigDto, + HlsCorruptSegmentWatchdogMode, HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, + HlsSegmentRepairConfigDto, HlsSegmentRepairMode, HlsStripConfigDto, HlsStripMode, QosAggregationConfigDto, + RateLimitConfigDto, ResourceRetryConfigDto, ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, + StreamBufferConfigDto, StreamConfigDto, StreamHistoryConfigDto, }, - utils::{default_secret, format_float_localized}, + utils::format_float_localized, }; use std::{rc::Rc, str::FromStr}; use strum::IntoEnumIterator; +use web_sys::HtmlInputElement; use yew::prelude::*; const LABEL_CACHE: &str = "LABEL.CACHE"; +const LABEL_RESOURCE_IMAGE_CACHE: &str = "LABEL.RESOURCE_IMAGE_CACHE"; const LABEL_ENABLED: &str = "LABEL.ENABLED"; const LABEL_SIZE: &str = "LABEL.SIZE"; const LABEL_DIRECTORY: &str = "LABEL.DIRECTORY"; @@ -52,6 +60,29 @@ const LABEL_THROTTLE_KBPS: &str = "LABEL.THROTTLE_KBPS"; const LABEL_STREAM_BUFFER: &str = "LABEL.STREAM_BUFFER"; const LABEL_BUFFER_ENABLED: &str = "LABEL.BUFFER_ENABLED"; const LABEL_BUFFER_SIZE: &str = "LABEL.BUFFER_SIZE"; +const LABEL_HLS_CACHE_PROXY: &str = "LABEL.HLS_CACHE_PROXY"; +const LABEL_HLS_CACHE_SEGMENT_REPAIR: &str = "LABEL.HLS_CACHE_SEGMENT_REPAIR"; +const LABEL_CACHE_PATH: &str = "LABEL.CACHE_PATH"; +const LABEL_STRIP_MODE: &str = "LABEL.STRIP_MODE"; +const LABEL_STRIP_VALUE: &str = "LABEL.STRIP_VALUE"; +const LABEL_CACHE_DURATION: &str = "LABEL.CACHE_DURATION"; +const LABEL_CACHE_BYTES: &str = "LABEL.CACHE_BYTES"; +const LABEL_CACHE_BYTES_PER_SESSION: &str = "LABEL.CACHE_BYTES_PER_SESSION"; +const LABEL_MAX_SEGMENTS_PREFETCH: &str = "LABEL.MAX_SEGMENTS_PREFETCH"; +const LABEL_MAX_CONCURRENT_SEGMENT_FETCHES_PER_SESSION: &str = "LABEL.MAX_CONCURRENT_SEGMENT_FETCHES_PER_SESSION"; +const LABEL_MAX_CONCURRENT_SEGMENT_FETCHES_GLOBAL: &str = "LABEL.MAX_CONCURRENT_SEGMENT_FETCHES_GLOBAL"; +const LABEL_ORIGIN_MANIFEST_TIMEOUT_MS: &str = "LABEL.ORIGIN_MANIFEST_TIMEOUT_MS"; +const LABEL_MANIFEST_RECOVERY_BURST: &str = "LABEL.MANIFEST_RECOVERY_BURST"; +const LABEL_ORIGIN_SEGMENT_TIMEOUT_MS: &str = "LABEL.ORIGIN_SEGMENT_TIMEOUT_MS"; +const LABEL_SESSION_IDLE_TIMEOUT: &str = "LABEL.SESSION_IDLE_TIMEOUT"; +const LABEL_SEGMENT_REPAIR: &str = "LABEL.SEGMENT_REPAIR"; +const LABEL_SEGMENT_SIZE_INCREASE: &str = "LABEL.SEGMENT_SIZE_INCREASE"; +const LABEL_REPAIR_TRIGGER: &str = "LABEL.REPAIR_TRIGGER"; +const LABEL_APPLY_TO_FIRST_SEGMENTS: &str = "LABEL.APPLY_TO_FIRST_SEGMENTS"; +const LABEL_MAX_PARALLEL_REPAIRS: &str = "LABEL.MAX_PARALLEL_REPAIRS"; +const LABEL_POSTPROCESS_TIMEOUT_MS: &str = "LABEL.POSTPROCESS_TIMEOUT_MS"; +const LABEL_CORRUPT_SEGMENT_WATCHDOG: &str = "LABEL.CORRUPT_SEGMENT_WATCHDOG"; +const LABEL_MAX_WATCHDOG_JOBS: &str = "LABEL.MAX_WATCHDOG_JOBS"; const LABEL_RATE_LIMIT: &str = "LABEL.RATE_LIMIT"; const LABEL_PERIOD_MILLIS: &str = "LABEL.PERIOD_MILLIS"; @@ -92,7 +123,7 @@ generate_form_reducer!( action_name: CacheConfigFormAction, fields { Enabled => enabled: bool, - Size => size: Option, + Size => size: Option, Dir => directory: Option, } ); @@ -179,6 +210,34 @@ generate_form_reducer!( } ); +generate_form_reducer!( + state: StripConfigFormState { form: HlsStripConfigDto }, + action_name: StripConfigFormAction, + fields { + Mode => mode: HlsStripMode, + Value => value: u64, + } +); + +generate_form_reducer!( + state: HlsCacheConfigFormState { form: HlsCacheConfigDto }, + action_name: HlsCacheConfigFormAction, + fields { + CachePath => cache_path: Option, + CacheDuration => cache_duration: u64, + CacheBytes => cache_bytes: ByteSize, + CacheBytesPerSession => cache_bytes_per_session: ByteSize, + MaxSegmentsPrefetch => max_segments_prefetch: usize, + MaxConcurrentSegmentFetchesPerSession => max_concurrent_segment_fetches_per_session: usize, + MaxConcurrentSegmentFetchesGlobal => max_concurrent_segment_fetches_global: usize, + OriginManifestTimeoutMs => origin_manifest_timeout_ms: u64, + ManifestRecoveryBurst => manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto, + OriginSegmentTimeoutMs => origin_segment_timeout_ms: u64, + SessionIdleTimeout => session_idle_timeout: u64, + SegmentRepair => segment_repair: HlsSegmentRepairConfigDto, + } +); + generate_form_reducer!( state: StreamHistoryConfigFormState { form: StreamHistoryConfigDto }, action_name: StreamHistoryConfigFormAction, @@ -237,6 +296,122 @@ fn geoip_unavailable_policy_labels(translate: &YewI18n) -> Rc> { ]) } +fn hls_strip_mode_options(selected: HlsStripMode) -> Rc> { + Rc::new(vec![ + DropDownOption::new("segments", html! { "segments" }, selected == HlsStripMode::Segments), + DropDownOption::new("seconds", html! { "seconds" }, selected == HlsStripMode::Seconds), + ]) +} + +fn hls_manifest_recovery_burst_options(selected: HlsManifestRecoveryBurstLevel) -> Rc> { + Rc::new(vec![ + DropDownOption::new("off", html! { "OFF" }, selected == HlsManifestRecoveryBurstLevel::Off), + DropDownOption::new("friendly", html! { "FRIENDLY" }, selected == HlsManifestRecoveryBurstLevel::Friendly), + DropDownOption::new("cautious", html! { "CAUTIOUS" }, selected == HlsManifestRecoveryBurstLevel::Cautious), + DropDownOption::new("balanced", html! { "BALANCED" }, selected == HlsManifestRecoveryBurstLevel::Balanced), + DropDownOption::new("intense", html! { "INTENSE" }, selected == HlsManifestRecoveryBurstLevel::Intense), + DropDownOption::new( + "aggressive", + html! { "AGGRESSIVE" }, + selected == HlsManifestRecoveryBurstLevel::Aggressive, + ), + DropDownOption::new("beast", html! { "BEAST" }, selected == HlsManifestRecoveryBurstLevel::Beast), + ]) +} + +fn hls_manifest_recovery_burst_label(level: HlsManifestRecoveryBurstLevel) -> &'static str { + match level { + HlsManifestRecoveryBurstLevel::Off => "OFF", + HlsManifestRecoveryBurstLevel::Friendly => "FRIENDLY", + HlsManifestRecoveryBurstLevel::Cautious => "CAUTIOUS", + HlsManifestRecoveryBurstLevel::Balanced => "BALANCED", + HlsManifestRecoveryBurstLevel::Intense => "INTENSE", + HlsManifestRecoveryBurstLevel::Aggressive => "AGGRESSIVE", + HlsManifestRecoveryBurstLevel::Beast => "BEAST", + } +} + +fn hls_segment_repair_max_level_options(selected: HlsSegmentRepairMode) -> Rc> { + Rc::new(vec![ + DropDownOption::new("off", html! { "OFF" }, selected == HlsSegmentRepairMode::Off), + DropDownOption::new("low", html! { "LOW" }, selected == HlsSegmentRepairMode::Low), + DropDownOption::new("medium", html! { "MEDIUM" }, selected == HlsSegmentRepairMode::Medium), + DropDownOption::new("high", html! { "HIGH" }, selected == HlsSegmentRepairMode::High), + ]) +} + +fn hls_segment_repair_max_level_label(mode: HlsSegmentRepairMode) -> &'static str { + match mode { + HlsSegmentRepairMode::Off => "OFF", + HlsSegmentRepairMode::Low => "LOW", + HlsSegmentRepairMode::Medium => "MEDIUM", + HlsSegmentRepairMode::High => "HIGH", + } +} + +fn hls_corrupt_segment_watchdog_options(selected: HlsCorruptSegmentWatchdogMode) -> Rc> { + Rc::new(vec![ + DropDownOption::new("off", html! { "OFF" }, selected == HlsCorruptSegmentWatchdogMode::Off), + DropDownOption::new( + "detect_only", + html! { "DETECT ONLY" }, + selected == HlsCorruptSegmentWatchdogMode::DetectOnly, + ), + DropDownOption::new("sanitize", html! { "SANITIZE" }, selected == HlsCorruptSegmentWatchdogMode::Sanitize), + DropDownOption::new( + "diagnostic", + html! { "DIAGNOSTIC" }, + selected == HlsCorruptSegmentWatchdogMode::Diagnostic, + ), + ]) +} + +fn hls_corrupt_segment_watchdog_label(mode: HlsCorruptSegmentWatchdogMode) -> &'static str { + match mode { + HlsCorruptSegmentWatchdogMode::Off => "OFF", + HlsCorruptSegmentWatchdogMode::DetectOnly => "DETECT ONLY", + HlsCorruptSegmentWatchdogMode::Sanitize => "SANITIZE", + HlsCorruptSegmentWatchdogMode::Diagnostic => "DIAGNOSTIC", + } +} + +fn hls_segment_repair_size_increase_percent(segment_repair: &HlsSegmentRepairConfigDto) -> Option { + match segment_repair.max_level { + HlsSegmentRepairMode::Off => None, + HlsSegmentRepairMode::Low => Some(segment_repair.size_increase.low_percent), + HlsSegmentRepairMode::Medium => Some(segment_repair.size_increase.medium_percent), + HlsSegmentRepairMode::High => Some(segment_repair.size_increase.high_percent), + } +} + +fn set_hls_segment_repair_size_increase_percent(segment_repair: &mut HlsSegmentRepairConfigDto, value: u8) { + match segment_repair.max_level { + HlsSegmentRepairMode::Off => {} + HlsSegmentRepairMode::Low => segment_repair.size_increase.low_percent = value, + HlsSegmentRepairMode::Medium => segment_repair.size_increase.medium_percent = value, + HlsSegmentRepairMode::High => segment_repair.size_increase.high_percent = value, + } +} + +fn hls_segment_repair_size_increase_label(translate: &YewI18n, mode: HlsSegmentRepairMode) -> String { + match mode { + HlsSegmentRepairMode::Off => translate.t(LABEL_SEGMENT_SIZE_INCREASE), + _ => format!("{} ({})", translate.t(LABEL_SEGMENT_SIZE_INCREASE), hls_segment_repair_max_level_label(mode)), + } +} + +fn clamp_u64_min(value: Option, min_value: u64) -> u64 { + value.and_then(|value| u64::try_from(value).ok()).filter(|value| *value >= min_value).unwrap_or(min_value) +} + +fn clamp_usize_min(value: Option, min_value: usize) -> usize { + value.and_then(|value| usize::try_from(value).ok()).filter(|value| *value >= min_value).unwrap_or(min_value) +} + +fn clamp_u8_range(value: Option, min_value: u8, max_value: u8) -> u8 { + value.and_then(|value| u8::try_from(value).ok()).map(|value| value.clamp(min_value, max_value)).unwrap_or(min_value) +} + #[component] pub fn ReverseProxyConfigView() -> Html { let translate = use_translation(); @@ -264,6 +439,10 @@ pub fn ReverseProxyConfigView() -> Html { let geoip_state: UseReducerHandle = use_reducer(|| GeoIpConfigFormState { form: GeoIpConfigDto::default(), modified: false }); + let hls_cache_state: UseReducerHandle = + use_reducer(|| HlsCacheConfigFormState { form: HlsCacheConfigDto::default(), modified: false }); + let hls_strip_state: UseReducerHandle = + use_reducer(|| StripConfigFormState { form: HlsStripConfigDto::default(), modified: false }); let stream_buffer_state: UseReducerHandle = use_reducer(|| StreamBufferConfigFormState { form: StreamBufferConfigDto::default(), modified: false }); @@ -286,6 +465,8 @@ pub fn ReverseProxyConfigView() -> Html { let resource_retry_state = resource_retry_state.clone(); let stream_state = stream_state.clone(); let geoip_state = geoip_state.clone(); + let hls_cache_state = hls_cache_state.clone(); + let hls_strip_state = hls_strip_state.clone(); let stream_buffer_state = stream_buffer_state.clone(); let failover_patterns_state = failover_patterns_state.clone(); let admission_strategies_state = admission_strategies_state.clone(); @@ -296,63 +477,79 @@ pub fn ReverseProxyConfigView() -> Html { use_emit_mapped_option( ( ( - reverse_proxy_state.form.clone(), - disabled_header_state.form.clone(), - cache_state.form.clone(), - rate_limit_state.form.clone(), - resource_retry_state.form.clone(), - stream_state.form.clone(), - geoip_state.form.clone(), - stream_buffer_state.form.clone(), - failover_patterns_state.form.clone(), - admission_strategies_state.form.clone(), - stream_history_state.form.clone(), - qos_aggregation_state.form.clone(), + ( + reverse_proxy_state.form.clone(), + disabled_header_state.form.clone(), + cache_state.form.clone(), + rate_limit_state.form.clone(), + resource_retry_state.form.clone(), + stream_state.form.clone(), + geoip_state.form.clone(), + ), + ( + hls_cache_state.form.clone(), + hls_strip_state.form.clone(), + stream_buffer_state.form.clone(), + failover_patterns_state.form.clone(), + admission_strategies_state.form.clone(), + stream_history_state.form.clone(), + qos_aggregation_state.form.clone(), + ), ), ( - reverse_proxy_state.modified, - disabled_header_state.modified, - cache_state.modified, - rate_limit_state.modified, - resource_retry_state.modified, - stream_state.modified, - geoip_state.modified, - stream_buffer_state.modified, - failover_patterns_state.modified, - admission_strategies_state.modified, - stream_history_state.modified, - qos_aggregation_state.modified, + ( + reverse_proxy_state.modified, + disabled_header_state.modified, + cache_state.modified, + rate_limit_state.modified, + resource_retry_state.modified, + stream_state.modified, + geoip_state.modified, + ), + ( + hls_cache_state.modified, + hls_strip_state.modified, + stream_buffer_state.modified, + failover_patterns_state.modified, + admission_strategies_state.modified, + stream_history_state.modified, + qos_aggregation_state.modified, + ), ), ), config_view_ctx.on_form_change.clone(), move |( ( - rp, - disabled_header, - cache, - rl, - resource_retry, - stream, - geoip, - stream_buffer, - failover_patterns, - admission_strategies, - stream_history, - qos_aggregation, + (rp, disabled_header, cache, rl, resource_retry, stream, geoip), + ( + hls_cache, + hls_strip, + stream_buffer, + failover_patterns, + admission_strategies, + stream_history, + qos_aggregation, + ), ), ( - rp_modified, - disabled_header_modified, - cache_modified, - rl_modified, - resource_retry_modified, - stream_modified, - geoip_modified, - stream_buffer_modified, - failover_patterns_modified, - admission_strategies_modified, - stream_history_modified, - qos_aggregation_modified, + ( + rp_modified, + disabled_header_modified, + cache_modified, + rl_modified, + resource_retry_modified, + stream_modified, + geoip_modified, + ), + ( + hls_cache_modified, + hls_strip_modified, + stream_buffer_modified, + failover_patterns_modified, + admission_strategies_modified, + stream_history_modified, + qos_aggregation_modified, + ), ), )| { let mut form = rp.clone(); @@ -371,6 +568,9 @@ pub fn ReverseProxyConfigView() -> Html { form.resource_retry = Some(resource_retry_form); form.stream = Some(stream_form); form.geoip = Some(geoip.clone()); + let mut hls_cache_form = hls_cache.clone(); + hls_cache_form.strip = hls_strip.clone(); + form.hls_cache = Some(hls_cache_form); form.disabled_header = if disabled_header.is_empty() { None } else { Some(disabled_header.clone()) }; form.stream_history = if stream_history.is_empty() { None } else { Some(stream_history.clone()) }; form.qos_aggregation = if qos_aggregation.is_empty() { None } else { Some(qos_aggregation.clone()) }; @@ -382,6 +582,7 @@ pub fn ReverseProxyConfigView() -> Html { || resource_retry_modified || stream_modified || geoip_modified + || (hls_cache_modified || hls_strip_modified) || stream_buffer_modified || failover_patterns_modified || admission_strategies_modified @@ -407,6 +608,8 @@ pub fn ReverseProxyConfigView() -> Html { let resource_retry_state = resource_retry_state.clone(); let stream_state = stream_state.clone(); let geoip_state = geoip_state.clone(); + let hls_cache_state = hls_cache_state.clone(); + let hls_strip_state = hls_strip_state.clone(); let stream_buffer_state = stream_buffer_state.clone(); let failover_patterns_state = failover_patterns_state.clone(); let admission_strategies_state = admission_strategies_state.clone(); @@ -456,6 +659,14 @@ pub fn ReverseProxyConfigView() -> Html { geoip_state.dispatch(GeoIpConfigFormAction::SetAll(target_geoip)); } + let target_hls_cache = rp.hls_cache.as_ref().map_or_else(HlsCacheConfigDto::default, |h| h.clone()); + if hls_cache_state.form != target_hls_cache { + hls_cache_state.dispatch(HlsCacheConfigFormAction::SetAll(target_hls_cache.clone())); + } + if hls_strip_state.form != target_hls_cache.strip { + hls_strip_state.dispatch(StripConfigFormAction::SetAll(target_hls_cache.strip)); + } + let target_stream_buffer = rp.stream.as_ref().and_then(|s| s.buffer.clone()).unwrap_or_default(); if stream_buffer_state.form != target_stream_buffer { stream_buffer_state.dispatch(StreamBufferConfigFormAction::SetAll(target_stream_buffer)); @@ -530,6 +741,16 @@ pub fn ReverseProxyConfigView() -> Html { geoip_state.dispatch(GeoIpConfigFormAction::SetAll(target_geoip)); } + let target_hls_cache = HlsCacheConfigDto::default(); + if hls_cache_state.form != target_hls_cache { + hls_cache_state.dispatch(HlsCacheConfigFormAction::SetAll(target_hls_cache)); + } + + let target_hls_strip = HlsStripConfigDto::default(); + if hls_strip_state.form != target_hls_strip { + hls_strip_state.dispatch(StripConfigFormAction::SetAll(target_hls_strip)); + } + let target_stream_buffer = StreamBufferConfigDto::default(); if stream_buffer_state.form != target_stream_buffer { stream_buffer_state.dispatch(StreamBufferConfigFormAction::SetAll(target_stream_buffer)); @@ -563,13 +784,399 @@ pub fn ReverseProxyConfigView() -> Html { let render_cache = || { html! { -

{translate.t(LABEL_CACHE)}

+

{translate.t(LABEL_RESOURCE_IMAGE_CACHE)}

{ config_field_bool!(cache_state.form, translate.t(LABEL_ENABLED), enabled) } { config_field_optional!(cache_state.form, translate.t(LABEL_SIZE), size) } { config_field_optional!(cache_state.form, translate.t(LABEL_DIRECTORY), directory) } } }; + + let render_hls_cache = || { + html! { + +

{translate.t(LABEL_HLS_CACHE_PROXY)}

+ { config_field_optional!(hls_cache_state.form, translate.t(LABEL_CACHE_PATH), cache_path) } + { config_field_child!(translate.t(LABEL_STRIP_MODE), "HLS_CACHE_CONFIG.STRIP_MODE", { + html! { {hls_strip_state.form.mode.to_string()} } + }) } + { config_field_child!(translate.t(LABEL_STRIP_VALUE), "HLS_CACHE_CONFIG.STRIP_VALUE", { + html! { {hls_strip_state.form.value.to_string()} } + }) } + { config_field!(hls_cache_state.form, translate.t(LABEL_CACHE_DURATION), cache_duration) } + { config_field_child!(translate.t(LABEL_CACHE_BYTES), "HLS_CACHE_CONFIG.CACHE_BYTES", { + html! { {hls_cache_state.form.cache_bytes.as_str().to_string()} } + }) } + { config_field_child!(translate.t(LABEL_CACHE_BYTES_PER_SESSION), "HLS_CACHE_CONFIG.CACHE_BYTES_PER_SESSION", { + html! { {hls_cache_state.form.cache_bytes_per_session.as_str().to_string()} } + }) } + { config_field!(hls_cache_state.form, translate.t(LABEL_MAX_SEGMENTS_PREFETCH), max_segments_prefetch) } + { config_field!(hls_cache_state.form, translate.t(LABEL_MAX_CONCURRENT_SEGMENT_FETCHES_PER_SESSION), max_concurrent_segment_fetches_per_session) } + { config_field!(hls_cache_state.form, translate.t(LABEL_MAX_CONCURRENT_SEGMENT_FETCHES_GLOBAL), max_concurrent_segment_fetches_global) } + { config_field!(hls_cache_state.form, translate.t(LABEL_ORIGIN_MANIFEST_TIMEOUT_MS), origin_manifest_timeout_ms) } + { config_field_child!(translate.t(LABEL_MANIFEST_RECOVERY_BURST), "HLS_CACHE_CONFIG.MANIFEST_RECOVERY_BURST", { + html! { {hls_manifest_recovery_burst_label(hls_cache_state.form.manifest_recovery_burst.level)} } + }) } + { config_field!(hls_cache_state.form, translate.t(LABEL_ORIGIN_SEGMENT_TIMEOUT_MS), origin_segment_timeout_ms) } + { config_field!(hls_cache_state.form, translate.t(LABEL_SESSION_IDLE_TIMEOUT), session_idle_timeout) } + + } + }; + + let render_hls_segment_repair = || { + let segment_repair = &hls_cache_state.form.segment_repair; + let watchdog = &segment_repair.corrupt_segment_watchdog; + let size_increase = hls_segment_repair_size_increase_percent(segment_repair) + .map_or_else(|| "-".to_string(), |value| format!("{value}%")); + html! { + +

{translate.t(LABEL_HLS_CACHE_SEGMENT_REPAIR)}

+ { config_field_child!(translate.t(LABEL_SEGMENT_REPAIR), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_MAX_LEVEL", { + html! { {hls_segment_repair_max_level_label(segment_repair.max_level)} } + }) } + { config_field_child!(translate.t(LABEL_APPLY_TO_FIRST_SEGMENTS), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_APPLY_TO_FIRST_SEGMENTS", { + html! { {segment_repair.apply_to_first_segments.to_string()} } + }) } + { config_field_child!(translate.t(LABEL_MAX_PARALLEL_REPAIRS), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_MAX_PARALLEL_REPAIRS", { + html! { {segment_repair.max_parallel_repairs.to_string()} } + }) } + { config_field_child!(translate.t(LABEL_POSTPROCESS_TIMEOUT_MS), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_POSTPROCESS_TIMEOUT_MS", { + html! { {segment_repair.postprocess_timeout_ms.to_string()} } + }) } + { config_field_child!(hls_segment_repair_size_increase_label(&translate, segment_repair.max_level), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_SIZE_INCREASE", { + html! { {size_increase} } + }) } + { config_field_child!(translate.t(LABEL_REPAIR_TRIGGER), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_TRIGGER", { + html! { {"automatic codec trigger policy"} } + }) } + { config_field_child!(translate.t(LABEL_CORRUPT_SEGMENT_WATCHDOG), "HLS_CACHE_CONFIG.CORRUPT_SEGMENT_WATCHDOG", { + html! { {hls_corrupt_segment_watchdog_label(watchdog.mode)} } + }) } + { config_field_child!(translate.t(LABEL_MAX_WATCHDOG_JOBS), "HLS_CACHE_CONFIG.CORRUPT_SEGMENT_WATCHDOG_MAX_PARALLEL_JOBS", { + html! { {watchdog.max_parallel_jobs.to_string()} } + }) } + + } + }; + + let render_hls_cache_edit = || { + let selected_strip_mode = hls_strip_state.form.mode; + let strip_state = hls_strip_state.clone(); + let set_max_segments_prefetch = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |value: Option| { + let max_segments_prefetch = value.and_then(|value| usize::try_from(value).ok()).unwrap_or(0); + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.max_parallel_repairs = segment_repair.max_parallel_repairs.min(max_segments_prefetch); + segment_repair.corrupt_segment_watchdog.max_parallel_jobs = + segment_repair.corrupt_segment_watchdog.max_parallel_jobs.min(max_segments_prefetch.max(1)); + hls_cache_state.dispatch(HlsCacheConfigFormAction::MaxSegmentsPrefetch(max_segments_prefetch)); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }) + }; + let edit_hls_cache_u64_min = + |label: String, field: &'static str, value: u64, action: fn(u64) -> HlsCacheConfigFormAction| { + let hls_cache_state = hls_cache_state.clone(); + html! { +
+ | { + hls_cache_state.dispatch(action(clamp_u64_min(value, 1))); + })} + /> +
+ } + }; + let edit_hls_cache_usize_min = + |label: String, field: &'static str, value: usize, action: fn(usize) -> HlsCacheConfigFormAction| { + let hls_cache_state = hls_cache_state.clone(); + html! { +
+ | { + hls_cache_state.dispatch(action(clamp_usize_min(value, 1))); + })} + /> +
+ } + }; + let selected_manifest_recovery_burst = hls_cache_state.form.manifest_recovery_burst.level; + let set_manifest_recovery_burst = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |(_, selections): (String, DropDownSelection)| { + if let DropDownSelection::Single(selection) = selections { + if let Ok(level) = HlsManifestRecoveryBurstLevel::from_str(selection.as_str()) { + hls_cache_state.dispatch(HlsCacheConfigFormAction::ManifestRecoveryBurst( + HlsManifestRecoveryBurstConfigDto { level }, + )); + } + } + }) + }; + html! { + +

{translate.t(LABEL_HLS_CACHE_PROXY)}

+ { edit_field_text_option!(hls_cache_state, translate.t(LABEL_CACHE_PATH), cache_path, HlsCacheConfigFormAction::CachePath) } + { config_field_child!(translate.t(LABEL_STRIP_MODE), "HLS_CACHE_CONFIG.STRIP_MODE", { + html! { + + } + }) } + { edit_hls_cache_u64_min(translate.t(LABEL_ORIGIN_SEGMENT_TIMEOUT_MS), "origin_segment_timeout_ms", hls_cache_state.form.origin_segment_timeout_ms, HlsCacheConfigFormAction::OriginSegmentTimeoutMs) } + { edit_hls_cache_u64_min(translate.t(LABEL_SESSION_IDLE_TIMEOUT), "session_idle_timeout", hls_cache_state.form.session_idle_timeout, HlsCacheConfigFormAction::SessionIdleTimeout) } + + } + }; + + let render_hls_segment_repair_edit = || { + let segment_repair = hls_cache_state.form.segment_repair.clone(); + let watchdog = segment_repair.corrupt_segment_watchdog.clone(); + let selected_segment_repair_max_level = segment_repair.max_level; + let set_segment_repair_max_level = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |(_, selections): (String, DropDownSelection)| { + if let DropDownSelection::Single(selection) = selections { + if let Ok(mode) = HlsSegmentRepairMode::from_str(selection.as_str()) { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.max_level = mode; + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + } + } + }) + }; + let set_segment_repair_apply_to_first_segments = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |value: Option| { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.apply_to_first_segments = clamp_u8_range(value, 0, 6); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }) + }; + let set_segment_repair_max_parallel_repairs = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |value: Option| { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.max_parallel_repairs = + clamp_usize_min(value, 1).min(hls_cache_state.form.max_segments_prefetch); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }) + }; + let set_segment_repair_postprocess_timeout_ms = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |value: Option| { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.postprocess_timeout_ms = clamp_u64_min(value, 100); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }) + }; + let selected_watchdog_mode = watchdog.mode; + let set_watchdog_mode = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |(_, selections): (String, DropDownSelection)| { + if let DropDownSelection::Single(selection) = selections { + if let Ok(mode) = HlsCorruptSegmentWatchdogMode::from_str(selection.as_str()) { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.corrupt_segment_watchdog.mode = mode; + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + } + } + }) + }; + let set_watchdog_max_parallel_jobs = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |value: Option| { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + segment_repair.corrupt_segment_watchdog.max_parallel_jobs = + clamp_usize_min(value, 1).min(hls_cache_state.form.max_segments_prefetch.max(1)); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }) + }; + let render_slider_control = |name: String, value: u8, max_value: u8, on_change: Callback| -> Html { + let value_string = value.to_string(); + let max_string = max_value.to_string(); + let fill = if max_value == 0 { 0 } else { (u16::from(value) * 100) / u16::from(max_value) }; + let slider_style = format!("--tp-hls-repair-slider-fill: {fill}%;"); + let oninput = Callback::from(move |event: InputEvent| { + let input: HtmlInputElement = event.target_unchecked_into(); + if let Ok(value) = input.value().parse::() { + on_change.emit(value.min(max_value)); + } + }); + html! { +
+ + {value_string} +
+ } + }; + let render_slider = + |label: String, info_key: &'static str, name: String, value: u8, max_value: u8, on_change: Callback| { + let control = render_slider_control(name, value, max_value, on_change); + config_field_child!(label, info_key, { + html! { + { control } + } + }) + }; + let size_increase_editor = + if let Some(size_increase_percent) = hls_segment_repair_size_increase_percent(&segment_repair) { + let hls_cache_state = hls_cache_state.clone(); + render_slider( + hls_segment_repair_size_increase_label(&translate, segment_repair.max_level), + "HLS_CACHE_CONFIG.SEGMENT_REPAIR_SIZE_INCREASE", + "hls_segment_repair_size_increase".to_string(), + size_increase_percent, + 100, + Callback::from(move |value| { + let mut segment_repair = hls_cache_state.form.segment_repair.clone(); + set_hls_segment_repair_size_increase_percent(&mut segment_repair, value); + hls_cache_state.dispatch(HlsCacheConfigFormAction::SegmentRepair(segment_repair)); + }), + ) + } else { + config_field_child!( + translate.t(LABEL_SEGMENT_SIZE_INCREASE), + "HLS_CACHE_CONFIG.SEGMENT_REPAIR_SIZE_INCREASE", + { + html! { {"-"} } + } + ) + }; + + html! { + +

{translate.t(LABEL_HLS_CACHE_SEGMENT_REPAIR)}

+ { config_field_child!(translate.t(LABEL_SEGMENT_REPAIR), "HLS_CACHE_CONFIG.SEGMENT_REPAIR_MAX_LEVEL", { + html! { + + } + }) } +
+ +
+ + } + }; + let render_stream = || { let strategy_tags = displayed_admission_strategy_tags(&admission_strategies_state.form, &stream_state.form); html! { @@ -774,7 +1381,7 @@ pub fn ReverseProxyConfigView() -> Html {

{translate.t(LABEL_CACHE)}

{ edit_field_bool!(cache_state, translate.t(LABEL_ENABLED), enabled, CacheConfigFormAction::Enabled) } - { edit_field_text_option!(cache_state, translate.t(LABEL_SIZE), size, CacheConfigFormAction::Size) } + { edit_field_byte_size_option!(cache_state, translate.t(LABEL_SIZE), size, CacheConfigFormAction::Size) } { edit_field_text_option!(cache_state, translate.t(LABEL_DIRECTORY), directory, CacheConfigFormAction::Dir) } } @@ -960,6 +1567,8 @@ pub fn ReverseProxyConfigView() -> Html { { render_geoip() } { render_disabled_header_view() } { render_cache() } + { render_hls_cache() } + { render_hls_segment_repair() } { render_resource_retry_view() } { render_rate_limit() } { render_stream() } @@ -977,6 +1586,8 @@ pub fn ReverseProxyConfigView() -> Html { { render_geoip_edit() } { render_disabled_header_edit() } { render_cache_edit() } + { render_hls_cache_edit() } + { render_hls_segment_repair_edit() } { render_resource_retry_edit() } { render_rate_limit_edit() } { render_stream_edit() } diff --git a/frontend/src/app/components/dashboard/stream_display/helpers.rs b/frontend/src/app/components/dashboard/stream_display/helpers.rs index cd547cb57..d7329be00 100644 --- a/frontend/src/app/components/dashboard/stream_display/helpers.rs +++ b/frontend/src/app/components/dashboard/stream_display/helpers.rs @@ -1,12 +1,13 @@ use crate::{ app::ConfigContext, model::{BACKGROUND_TRANSFER_CLIENT_IP, BACKGROUND_TRANSFER_PROVIDER}, - utils::format_duration, + utils::{format_duration, is_shared_hls_stream}, }; use gloo_utils::window; use shared::{ + defaults::default_hls_session_ttl_secs, model::{PlaylistItemType, StreamChannel, StreamInfo, StreamInfoConfigDto, StreamTechnicalInfo}, - utils::{current_time_secs, default_hls_session_ttl_secs}, + utils::current_time_secs, }; use std::{ collections::{HashMap, HashSet}, @@ -105,7 +106,7 @@ fn compute_adaptive_last_seen( for stream in streams { if is_adaptive_session_stream(stream) { - if !stream.preserved || !next.contains_key(&stream.uid) { + if !stream.preserved || is_shared_hls_stream(stream) || !next.contains_key(&stream.uid) { next.insert(stream.uid, now); } } else { @@ -253,6 +254,12 @@ mod tests { }) } + fn test_shared_hls_stream(uid: u32, preserved: bool, has_session: bool) -> Rc { + let mut stream = (*test_stream(uid, PlaylistItemType::LiveHls, preserved, has_session)).clone(); + stream.channel.shared = true; + Rc::new(stream) + } + #[test] fn compute_adaptive_last_seen_prunes_missing_uids() { let existing = HashMap::from([(1, 100), (2, 200), (3, 300)]); @@ -270,4 +277,14 @@ mod tests { assert!(!refreshed.contains_key(&3)); assert!(!refreshed.contains_key(&9)); } + + #[test] + fn compute_adaptive_last_seen_refreshes_preserved_shared_hls() { + let existing = HashMap::from([(2, 200)]); + let streams = Some(vec![test_shared_hls_stream(2, true, true)]); + + let refreshed = compute_adaptive_last_seen(existing, &streams, 999); + + assert_eq!(refreshed.get(&2), Some(&999)); + } } diff --git a/frontend/src/app/components/dashboard/stream_display/meter.rs b/frontend/src/app/components/dashboard/stream_display/meter.rs index f062b7e67..aea9abdab 100644 --- a/frontend/src/app/components/dashboard/stream_display/meter.rs +++ b/frontend/src/app/components/dashboard/stream_display/meter.rs @@ -25,6 +25,7 @@ struct StreamMeterBadgeState { transferred_total_kb: u32, current_meter_uid: u32, current_meter_total_kb: u32, + has_sample: bool, } fn apply_stream_meter_entry( @@ -41,6 +42,7 @@ fn apply_stream_meter_entry( } next.current_meter_total_kb = next.current_meter_total_kb.max(entry.total_kb); next.rate_kbps = entry.rate_kbps; + next.has_sample = true; } else { next.transferred_total_kb = next.transferred_total_kb.saturating_add(entry.total_kb); } @@ -48,6 +50,24 @@ fn apply_stream_meter_entry( next } +fn format_meter_label(kind: MeterDisplayKind, state: &StreamMeterBadgeState) -> String { + if !state.has_sample { + return "-".to_string(); + } + match kind { + MeterDisplayKind::Bandwidth if state.rate_kbps == 0 => "0 KB/s".to_string(), + MeterDisplayKind::Bandwidth => format_bandwidth(state.rate_kbps), + MeterDisplayKind::Transferred => { + let total_kb = state.transferred_total_kb.saturating_add(state.current_meter_total_kb); + if total_kb == 0 { + "0 KB".to_string() + } else { + format_transferred(total_kb) + } + } + } +} + #[component] pub fn StreamMeterBadge(props: &StreamMeterBadgeProps) -> Html { let services = use_service_context(); @@ -55,7 +75,7 @@ pub fn StreamMeterBadge(props: &StreamMeterBadgeProps) -> Html { { let meter_state = meter_state.clone(); - let reset_key = props.uid; + let reset_key = props.meter_uid; use_effect_with(reset_key, move |_| { meter_state.set(StreamMeterBadgeState::default()); || () @@ -89,12 +109,43 @@ pub fn StreamMeterBadge(props: &StreamMeterBadgeProps) -> Html { }); } - let label = match props.kind { - MeterDisplayKind::Bandwidth => format_bandwidth(meter_state.rate_kbps), - MeterDisplayKind::Transferred => { - let total_kb = meter_state.transferred_total_kb.saturating_add(meter_state.current_meter_total_kb); - format_transferred(total_kb) - } - }; + let label = format_meter_label(props.kind, &meter_state); html! { {label} } } + +#[cfg(test)] +mod tests { + use super::{apply_stream_meter_entry, format_meter_label, MeterDisplayKind, StreamMeterBadgeState}; + use shared::model::StreamMeterEntry; + + fn entry(meter_uid: u32, rate_kbps: u32, total_kb: u32, uids: Vec) -> StreamMeterEntry { + StreamMeterEntry { meter_uid, rate_kbps, total_kb, uids } + } + + #[test] + fn meter_label_shows_dash_before_first_sample() { + let state = StreamMeterBadgeState::default(); + + assert_eq!(format_meter_label(MeterDisplayKind::Bandwidth, &state), "-"); + assert_eq!(format_meter_label(MeterDisplayKind::Transferred, &state), "-"); + } + + #[test] + fn meter_label_shows_zero_after_first_empty_sample() { + let state = apply_stream_meter_entry(&StreamMeterBadgeState::default(), 7, &entry(7, 0, 0, vec![1])); + + assert_eq!(format_meter_label(MeterDisplayKind::Bandwidth, &state), "0 KB/s"); + assert_eq!(format_meter_label(MeterDisplayKind::Transferred, &state), "0 KB"); + } + + #[test] + fn same_meter_keeps_transferred_total_across_stream_uid_changes() { + let state = apply_stream_meter_entry(&StreamMeterBadgeState::default(), 7, &entry(7, 120, 256, vec![1])); + let state = apply_stream_meter_entry(&state, 7, &entry(7, 0, 256, vec![2])); + + assert_eq!(state.current_meter_uid, 7); + assert_eq!(state.current_meter_total_kb, 256); + assert_eq!(state.transferred_total_kb, 0); + assert_eq!(format_meter_label(MeterDisplayKind::Transferred, &state), "256 KB"); + } +} diff --git a/frontend/src/app/components/dashboard/stream_display/mod.rs b/frontend/src/app/components/dashboard/stream_display/mod.rs index 60ae4b08c..a4d2db5a0 100644 --- a/frontend/src/app/components/dashboard/stream_display/mod.rs +++ b/frontend/src/app/components/dashboard/stream_display/mod.rs @@ -23,11 +23,11 @@ use crate::{ use gloo_timers::callback::Interval; pub use helpers::get_stream_info_config; use shared::{ + defaults::default_kick_secs, error::TuliproxError, model::{ PlaylistRequest, PlaylistUrlResolveRequest, ProtocolMessage, StreamInfo, StreamInfoConfigDto, UserCommand, }, - utils::default_kick_secs, }; use std::{collections::HashMap, fmt::Display, rc::Rc, str::FromStr}; use yew::{platform::spawn_local, prelude::*}; diff --git a/frontend/src/app/components/dashboard/stream_history_view.rs b/frontend/src/app/components/dashboard/stream_history_view.rs index d6bcf791d..ba66b2708 100644 --- a/frontend/src/app/components/dashboard/stream_history_view.rs +++ b/frontend/src/app/components/dashboard/stream_history_view.rs @@ -12,11 +12,11 @@ use crate::{ }; use futures::join; use shared::{ + defaults::default_page_size, model::{ PagedResponseDto, QosSnapshotRecordDto, QosSnapshotWindowDto, SearchRequest, StreamHistoryEventType, StreamHistoryPageRequestDto, StreamHistoryProviderSummaryDto, StreamHistoryRecordDto, }, - utils::default_page_size, }; use std::rc::Rc; use wasm_bindgen_futures::spawn_local; diff --git a/frontend/src/app/components/playlist/input/input_options.rs b/frontend/src/app/components/playlist/input/input_options.rs index 88cd58038..f7b66c6f6 100644 --- a/frontend/src/app/components/playlist/input/input_options.rs +++ b/frontend/src/app/components/playlist/input/input_options.rs @@ -3,8 +3,8 @@ use crate::{ i18n::use_translation, }; use shared::{ + defaults::{default_probe_delay_secs, default_probe_live_interval, default_resolve_delay_secs}, model::ConfigInputDto, - utils::{default_probe_delay_secs, default_probe_live_interval, default_resolve_delay_secs}, }; use std::rc::Rc; use yew::prelude::*; diff --git a/frontend/src/app/components/playlist/target/target_options.rs b/frontend/src/app/components/playlist/target/target_options.rs index fd0f5260f..d1f7017f0 100644 --- a/frontend/src/app/components/playlist/target/target_options.rs +++ b/frontend/src/app/components/playlist/target/target_options.rs @@ -22,6 +22,8 @@ pub fn TargetOptions(props: &TargetOptionsProps) -> Html { vec![ (false, "LABEL.IGNORE_LOGO"), (false, "LABEL.SHARE_LIVE_STREAMS"), + (false, "LABEL.HLS"), + (false, "LABEL.MPEG_TS"), (false, "LABEL.REMOVE_DUPLICATES"), ], redirect_default.clone(), @@ -38,7 +40,9 @@ pub fn TargetOptions(props: &TargetOptionsProps) -> Html { ( vec![ options.ignore_logo, - options.share_live_streams, + options.share_live_any_enabled(), + options.share_live_hls_enabled(), + options.share_live_mpeg_ts_enabled(), options.remove_duplicates, force_redirect[0].0, force_redirect[1].0, @@ -46,7 +50,9 @@ pub fn TargetOptions(props: &TargetOptionsProps) -> Html { ], vec![ (options.ignore_logo, "LABEL.IGNORE_LOGO"), - (options.share_live_streams, "LABEL.SHARE_LIVE_STREAMS"), + (options.share_live_any_enabled(), "LABEL.SHARE_LIVE_STREAMS"), + (options.share_live_hls_enabled(), "LABEL.HLS"), + (options.share_live_mpeg_ts_enabled(), "LABEL.MPEG_TS"), (options.remove_duplicates, "LABEL.REMOVE_DUPLICATES"), ], force_redirect, diff --git a/frontend/src/app/components/setup/setup_helpers.rs b/frontend/src/app/components/setup/setup_helpers.rs index e741ce10b..92c0f342a 100644 --- a/frontend/src/app/components/setup/setup_helpers.rs +++ b/frontend/src/app/components/setup/setup_helpers.rs @@ -3,12 +3,12 @@ use crate::app::{ ConfigContext, }; use shared::{ + defaults::{default_secret, is_default_supported_library_extensions}, model::{ ApiProxyConfigDto, AppConfigDto, HdHomeRunConfigDto, LibraryConfigDto, LibraryMetadataConfigDto, LibraryPlaylistConfigDto, LogConfigDto, ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, StreamConfigDto, TargetOutputDto, TargetUserDto, ThumbnailConfigDto, WebUiConfigDto, }, - utils::{default_secret, is_default_supported_library_extensions}, }; use std::sync::OnceLock; diff --git a/frontend/src/app/components/source_editor/target_form.rs b/frontend/src/app/components/source_editor/target_form.rs index f2138b166..d2d8945eb 100644 --- a/frontend/src/app/components/source_editor/target_form.rs +++ b/frontend/src/app/components/source_editor/target_form.rs @@ -2,6 +2,7 @@ use crate::{ app::components::{ config::HasFormData, select::Select, BlockId, BlockInstance, Card, ClusterFlagsInput, ClusterFlagsInputMode, DropDownOption, DropDownSelection, EditMode, FilterInput, IconButton, Panel, SourceEditorContext, TextButton, + ToggleSwitch, }, config_field, config_field_bool, config_field_child, config_field_custom, edit_field_bool, edit_field_list_option, edit_field_text, generate_form_reducer, @@ -9,7 +10,7 @@ use crate::{ }; use shared::{ error::TuliproxError, - model::{ClusterFlags, ConfigTargetDto, ConfigTargetOptions, ProcessingOrder}, + model::{ClusterFlags, ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, ProcessingOrder}, utils::Internable, }; use std::{fmt::Display, rc::Rc, str::FromStr, sync::Arc}; @@ -29,6 +30,8 @@ const LABEL_USE_MEMORY_CACHE: &str = "LABEL.USE_MEMORY_CACHE"; const LABEL_PROCESSING_ORDER: &str = "LABEL.PROCESSING_ORDER"; const LABEL_IGNORE_LOGO: &str = "LABEL.IGNORE_LOGO"; const LABEL_SHARE_LIVE_STREAMS: &str = "LABEL.SHARE_LIVE_STREAMS"; +const LABEL_HLS: &str = "LABEL.HLS"; +const LABEL_MPEG_TS: &str = "LABEL.MPEG_TS"; const LABEL_REMOVE_DUPLICATES: &str = "LABEL.REMOVE_DUPLICATES"; const LABEL_FORCE_REDIRECT: &str = "LABEL.FORCE_REDIRECT"; const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; @@ -80,16 +83,73 @@ impl Internable for TargetFormPage { // pub rename: Option>, // pub favourites: Option>, -generate_form_reducer!( - state: ConfigTargetOptionsFormState { form: ConfigTargetOptions }, - action_name: ConfigTargetOptionsFormAction, - fields { - IgnoreLogo => ignore_logo: bool, - ShareLiveStreams => share_live_streams: bool, - RemoveDuplicates => remove_duplicates: bool, - ForceRedirect => force_redirect: Option, +#[derive(Debug, Clone, PartialEq)] +pub struct ConfigTargetOptionsFormState { + pub form: ConfigTargetOptions, + modified: bool, +} + +impl HasFormData for ConfigTargetOptionsFormState { + type Data = ConfigTargetOptions; + + fn data(&self) -> &Self::Data { &self.form } + + fn modified(&self) -> bool { self.modified } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone)] +pub enum ConfigTargetOptionsFormAction { + IgnoreLogo(bool), + ShareLiveStreams(bool), + ShareLiveStreamsHls(bool), + ShareLiveStreamsMpegTs(bool), + RemoveDuplicates(bool), + ForceRedirect(Option), + SetAll(ConfigTargetOptions), +} + +impl yew::prelude::Reducible for ConfigTargetOptionsFormState { + type Action = ConfigTargetOptionsFormAction; + + fn reduce(self: Rc, action: Self::Action) -> Rc { + let mut form = self.form.clone(); + let modified; + + match action { + ConfigTargetOptionsFormAction::IgnoreLogo(value) => { + form.ignore_logo = value; + modified = true; + } + ConfigTargetOptionsFormAction::ShareLiveStreams(value) => { + form.share_live_streams = ConfigTargetShareLiveStreams { hls: value, mpeg_ts: value }; + modified = true; + } + ConfigTargetOptionsFormAction::ShareLiveStreamsHls(value) => { + form.share_live_streams.hls = value; + modified = true; + } + ConfigTargetOptionsFormAction::ShareLiveStreamsMpegTs(value) => { + form.share_live_streams.mpeg_ts = value; + modified = true; + } + ConfigTargetOptionsFormAction::RemoveDuplicates(value) => { + form.remove_duplicates = value; + modified = true; + } + ConfigTargetOptionsFormAction::ForceRedirect(value) => { + form.force_redirect = value; + modified = true; + } + ConfigTargetOptionsFormAction::SetAll(value) => { + form = value; + modified = false; + } + } + + Self { form, modified }.into() } -); +} generate_form_reducer!( state: ConfigTargetFormState { form: ConfigTargetDto }, @@ -171,39 +231,131 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { let render_options = || { let target_options_state_1 = target_options_state.clone(); + let target_option_toggle = + |label: String, field_id: &str, value: bool, readonly: bool, on_change: Callback| { + html! { +
+ + +
+ } + }; if !props.allow_write { html! { -
+
{ config_field_bool!(target_options_state.form, translate.t(LABEL_IGNORE_LOGO), ignore_logo) } - { config_field_bool!(target_options_state.form, translate.t(LABEL_SHARE_LIVE_STREAMS), share_live_streams) } +
+ { target_option_toggle( + translate.t(LABEL_SHARE_LIVE_STREAMS), + "CONFIG_TARGET_OPTIONS.SHARE_LIVE_STREAMS", + target_options_state.form.share_live_any_enabled(), + true, + Callback::noop(), + ) } +
+ { target_option_toggle( + translate.t(LABEL_HLS), + "CONFIG_TARGET_SHARE_LIVE_STREAMS.HLS", + target_options_state.form.share_live_hls_enabled(), + true, + Callback::noop(), + ) } + { target_option_toggle( + translate.t(LABEL_MPEG_TS), + "CONFIG_TARGET_SHARE_LIVE_STREAMS.MPEG_TS", + target_options_state.form.share_live_mpeg_ts_enabled(), + true, + Callback::noop(), + ) } +
+
+ { config_field_bool!(target_options_state.form, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates) } +
+
+ { translate.t(LABEL_FORCE_REDIRECT) } +
+
+
+ + { target_options_state.form.force_redirect.map_or_else(String::new, |flags| flags.to_string()) } + +
+
+
- { config_field_bool!(target_options_state.form, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates) } - { config_field_custom!( - translate.t(LABEL_FORCE_REDIRECT), - target_options_state.form.force_redirect.map_or_else(String::new, |flags| flags.to_string()) - ) } } } else { + let share_live_on_change = { + let target_options_state = target_options_state.clone(); + Callback::from(move |value| { + target_options_state.dispatch(ConfigTargetOptionsFormAction::ShareLiveStreams(value)); + }) + }; + let share_live_hls_on_change = { + let target_options_state = target_options_state.clone(); + Callback::from(move |value| { + target_options_state.dispatch(ConfigTargetOptionsFormAction::ShareLiveStreamsHls(value)); + }) + }; + let share_live_mpeg_ts_on_change = { + let target_options_state = target_options_state.clone(); + Callback::from(move |value| { + target_options_state.dispatch(ConfigTargetOptionsFormAction::ShareLiveStreamsMpegTs(value)); + }) + }; html! { -
- { edit_field_bool!(target_options_state, translate.t(LABEL_IGNORE_LOGO), ignore_logo, ConfigTargetOptionsFormAction::IgnoreLogo) } - { edit_field_bool!(target_options_state, translate.t(LABEL_SHARE_LIVE_STREAMS), share_live_streams, ConfigTargetOptionsFormAction::ShareLiveStreams) } +
+ { edit_field_bool!(target_options_state, translate.t(LABEL_IGNORE_LOGO), ignore_logo, ConfigTargetOptionsFormAction::IgnoreLogo) } +
+ { target_option_toggle( + translate.t(LABEL_SHARE_LIVE_STREAMS), + "CONFIG_TARGET_OPTIONS.SHARE_LIVE_STREAMS", + target_options_state.form.share_live_any_enabled(), + false, + share_live_on_change, + ) } +
+ { target_option_toggle( + translate.t(LABEL_HLS), + "CONFIG_TARGET_SHARE_LIVE_STREAMS.HLS", + target_options_state.form.share_live_hls_enabled(), + false, + share_live_hls_on_change, + ) } + { target_option_toggle( + translate.t(LABEL_MPEG_TS), + "CONFIG_TARGET_SHARE_LIVE_STREAMS.MPEG_TS", + target_options_state.form.share_live_mpeg_ts_enabled(), + false, + share_live_mpeg_ts_on_change, + ) } +
+
+ { edit_field_bool!(target_options_state, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates, ConfigTargetOptionsFormAction::RemoveDuplicates) } +
+
+ { translate.t(LABEL_FORCE_REDIRECT) } +
+
+
+ )| { + target_options_state_1.dispatch(ConfigTargetOptionsFormAction::ForceRedirect(flags)); + })} + /> +
+
+
- { edit_field_bool!(target_options_state, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates, ConfigTargetOptionsFormAction::RemoveDuplicates) } - { config_field_child!(translate.t(LABEL_FORCE_REDIRECT), "TARGET_FORM.FORCE_REDIRECT", { - html! { - )| { - target_options_state_1.dispatch(ConfigTargetOptionsFormAction::ForceRedirect(flags)); - })} - /> - }})} } } diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 90d0411f4..3f87dae4a 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -1,6 +1,7 @@ use crate::{ hooks::use_service_context, model::{EventMessage, BACKGROUND_TRANSFER_CLIENT_IP, BACKGROUND_TRANSFER_PROVIDER}, + utils::is_shared_hls_stream, }; use shared::{ model::{ @@ -53,7 +54,7 @@ fn dedupe_streams_by_identity(streams: &mut Vec) { } fn should_keep_preserved_stream_visible(stream: &StreamInfo) -> bool { - stream.session_token.is_some() && stream.channel.item_type.is_live_adaptive() + stream.session_token.is_some() && (stream.channel.item_type.is_live_adaptive() || is_shared_hls_stream(stream)) } fn should_keep_stream_when_connections_drop_to_zero(stream: &StreamInfo) -> bool { @@ -392,6 +393,12 @@ mod tests { } } + fn test_shared_hls_stream(uid: u32, addr: &str, session_token: Option<&str>) -> StreamInfo { + let mut stream = test_stream(uid, addr, session_token, PlaylistItemType::LiveHls); + stream.channel.shared = true; + stream + } + #[test] fn test_find_stream_update_index_prefers_adaptive_session_token_over_addr() { let existing = test_stream(1, "127.0.0.1:1234", Some("tok-hls"), PlaylistItemType::LiveHls); @@ -527,6 +534,22 @@ mod tests { assert_eq!(status.active_user_connections, 2); } + #[test] + fn test_preserved_shared_hls_update_keeps_stream_visible() { + let mut preserved = test_shared_hls_stream(1, "127.0.0.1:1234", Some("tok-hls")); + preserved.preserved = true; + let mut status = shared::model::StatusCheck { + active_users: 1, + active_user_connections: 1, + active_user_streams: vec![test_shared_hls_stream(1, "127.0.0.1:1234", Some("tok-hls"))], + ..Default::default() + }; + + apply_active_user_change(&mut status, ActiveUserConnectionChange::Updated(preserved.clone())); + + assert_eq!(status.active_user_streams, vec![preserved]); + } + #[test] fn test_disconnected_stream_removes_only_matching_stream_identity() { let removed = test_stream(1, "127.0.0.1:1234", Some("tok-live-a"), PlaylistItemType::Live); diff --git a/frontend/src/utils/mod.rs b/frontend/src/utils/mod.rs index e6c9aed5a..256e4d25a 100644 --- a/frontend/src/utils/mod.rs +++ b/frontend/src/utils/mod.rs @@ -3,6 +3,7 @@ mod storage; use crate::i18n::YewI18n; pub use format::*; +use shared::model::{PlaylistItemType, StreamInfo}; pub use storage::*; use wasm_bindgen::{prelude::Closure, JsCast}; use web_sys::window; @@ -62,3 +63,7 @@ pub fn join_non_empty_parts<'a>(parts: impl Iterator, separator: } result } + +pub fn is_shared_hls_stream(stream: &StreamInfo) -> bool { + stream.channel.shared && stream.channel.item_type == PlaylistItemType::LiveHls +} diff --git a/resources/hls_session_or_lease_expired.jpg b/resources/hls_session_or_lease_expired.jpg new file mode 100644 index 000000000..eb89c77a8 Binary files /dev/null and b/resources/hls_session_or_lease_expired.jpg differ diff --git a/shared/src/defaults/auth.rs b/shared/src/defaults/auth.rs new file mode 100644 index 000000000..a8227a931 --- /dev/null +++ b/shared/src/defaults/auth.rs @@ -0,0 +1,41 @@ +//! Auth, secret-generation and panel-API defaults. + +default_eq_fns!( + default_kick_secs, is_default_kick_secs, u64, 90; + default_token_ttl_mins, is_default_token_ttl_mins, u32, 30; + default_auth_error_status, is_default_auth_error_status, u16, 403; +); + +pub const fn default_panel_api_provision_timeout_secs() -> u64 { 65 } +pub const fn default_panel_api_provision_probe_interval_secs() -> u64 { 15 } +pub const fn default_panel_api_provision_cooldown_secs() -> u64 { 0 } +pub const fn default_panel_api_alias_pool_min() -> u16 { 1 } +pub const fn default_panel_api_alias_pool_max() -> u16 { 1 } + +fn fill_with_secure_random_bytes(out: &mut [u8]) { + #[cfg(target_arch = "wasm32")] + { + for byte in out { + *byte = fastrand::u8(..); + } + } + + #[cfg(not(target_arch = "wasm32"))] + if let Err(err) = getrandom::fill(out) { + panic!("failed to generate secure random bytes: {err}"); + } +} + +pub fn generate_default_access_secret() -> [u8; 32] { + let mut out = [0u8; 32]; + fill_with_secure_random_bytes(&mut out); + out +} + +pub fn generate_default_encrypt_secret() -> [u8; 16] { + let mut out = [0u8; 16]; + fill_with_secure_random_bytes(&mut out); + out +} + +pub fn default_secret() -> String { generate_default_encrypt_secret().iter().map(|b| format!("{:02X}", b)).collect() } diff --git a/shared/src/defaults/epg.rs b/shared/src/defaults/epg.rs new file mode 100644 index 000000000..1fc8b26ee --- /dev/null +++ b/shared/src/defaults/epg.rs @@ -0,0 +1,51 @@ +//! EPG defaults: match thresholds, normalize/strip patterns, episode pattern. + +default_eq_fns!( + default_epg_match_threshold, is_default_epg_match_threshold, u16, 80; + default_epg_best_match_threshold, is_default_epg_best_match_threshold, u16, 95; +); + +pub const DEFAULT_EPG_NORMALIZE_REGEX: &str = r"[^a-zA-Z0-9\-]"; + +pub fn default_epg_normalize_regex() -> Option { Some(DEFAULT_EPG_NORMALIZE_REGEX.to_string()) } +pub fn is_default_epg_normalize_regex(v: &Option) -> bool { + match v.as_ref().map(|value| value.trim()) { + None => true, + Some(value) => value.is_empty() || value == DEFAULT_EPG_NORMALIZE_REGEX, + } +} + +pub const DEFAULT_EPG_STRIP: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw", "full hd"]; +pub const DEFAULT_EPG_NAME_PREFIX_SEPARATOR: &[char] = &[':', '|', '-']; + +pub fn default_epg_strip() -> Option> { + Some(DEFAULT_EPG_STRIP.iter().map(|item| (*item).to_string()).collect()) +} +pub fn is_default_epg_strip(v: &Option>) -> bool { + let Some(current) = v.as_ref() else { + return true; + }; + let Some(default_strip) = default_epg_strip() else { + return false; + }; + current == &default_strip +} + +pub fn default_epg_name_prefix_separator() -> Option> { Some(DEFAULT_EPG_NAME_PREFIX_SEPARATOR.to_vec()) } +pub fn is_default_epg_name_prefix_separator(v: &Option>) -> bool { + let Some(current) = v.as_ref() else { + return true; + }; + let Some(default_separator) = default_epg_name_prefix_separator() else { + return false; + }; + current == &default_separator +} + +pub const DEFAULT_EPISODE_PATTERN: &str = r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*"; + +pub fn default_episode_pattern() -> Option { Some(DEFAULT_EPISODE_PATTERN.to_string()) } + +pub fn is_blank_or_default_episode_pattern(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || value.trim() == DEFAULT_EPISODE_PATTERN) +} diff --git a/shared/src/defaults/hdhomerun.rs b/shared/src/defaults/hdhomerun.rs new file mode 100644 index 000000000..a8be2ba27 --- /dev/null +++ b/shared/src/defaults/hdhomerun.rs @@ -0,0 +1,26 @@ +//! HDHomerun device defaults. + +const DEFAULT_FRIENDLY_NAME: &str = "TuliproxTV"; +const DEFAULT_MANUFACTURER: &str = "Silicondust"; +const DEFAULT_MODEL_NAME: &str = "HDTC-2US"; +const DEFAULT_FIRMWARE_NAME: &str = "hdhomeruntc_atsc"; +const DEFAULT_FIRMWARE_VERSION: &str = "20170930"; +const DEFAULT_DEVICE_TYPE: &str = "urn:schemas-upnp-org:device:MediaServer:1"; +const DEFAULT_DEVICE_UDN: &str = + "uuid:12345678-90ab-cdef-1234-567890abcdef::urn:dial-multicast:com.silicondust.hdhomerun"; + +pub fn default_friendly_name() -> String { DEFAULT_FRIENDLY_NAME.into() } +pub fn default_manufacturer() -> String { DEFAULT_MANUFACTURER.into() } +pub fn default_model_name() -> String { DEFAULT_MODEL_NAME.into() } +pub fn default_firmware_name() -> String { DEFAULT_FIRMWARE_NAME.into() } +pub fn default_firmware_version() -> String { DEFAULT_FIRMWARE_VERSION.into() } +pub fn default_device_type() -> String { DEFAULT_DEVICE_TYPE.into() } +pub fn default_device_udn() -> String { DEFAULT_DEVICE_UDN.into() } + +pub fn is_default_friendly_name(value: &String) -> bool { value == DEFAULT_FRIENDLY_NAME } +pub fn is_default_manufacturer(value: &String) -> bool { value == DEFAULT_MANUFACTURER } +pub fn is_default_model_name(value: &String) -> bool { value == DEFAULT_MODEL_NAME } +pub fn is_default_firmware_name(value: &String) -> bool { value == DEFAULT_FIRMWARE_NAME } +pub fn is_default_firmware_version(value: &String) -> bool { value == DEFAULT_FIRMWARE_VERSION } +pub fn is_default_device_type(value: &String) -> bool { value == DEFAULT_DEVICE_TYPE } +pub fn is_default_device_udn(value: &String) -> bool { value == DEFAULT_DEVICE_UDN } diff --git a/shared/src/defaults/hls.rs b/shared/src/defaults/hls.rs new file mode 100644 index 000000000..7a39f539e --- /dev/null +++ b/shared/src/defaults/hls.rs @@ -0,0 +1,49 @@ +//! HLS-cache defaults: path, size budgets, timeouts, segment-repair knobs, +//! corrupt-segment watchdog, session TTLs, manifest-recovery burst sizes. + +use crate::model::ByteSize; + +default_eq_fns!( + default_hls_session_ttl_secs, is_default_hls_session_ttl_secs, u64, 15; +); + +// Resolved against the OS temp dir so the default works on Linux, macOS, and Windows. +pub const HLS_CACHE_DIR_SUFFIX: &str = "tuliprox/cache/hls"; +pub const DEFAULT_HLS_CACHE_BYTES: &str = "10GB"; +pub const DEFAULT_HLS_CACHE_BYTES_PER_SESSION: &str = "512MB"; + +pub const fn default_hls_cache_duration() -> u64 { 300 } +pub fn default_hls_cache_bytes() -> ByteSize { ByteSize::new(DEFAULT_HLS_CACHE_BYTES) } +pub fn default_hls_cache_bytes_per_session() -> ByteSize { ByteSize::new(DEFAULT_HLS_CACHE_BYTES_PER_SESSION) } +pub const fn default_hls_max_segments_prefetch() -> usize { 6 } +pub const fn default_hls_max_concurrent_segment_fetches_per_session() -> usize { 2 } +pub const fn default_hls_max_concurrent_segment_fetches_global() -> usize { 64 } +pub const fn default_hls_origin_manifest_timeout_ms() -> u64 { 3_000 } +pub const fn default_hls_origin_segment_timeout_ms() -> u64 { 10_000 } +pub const fn default_hls_session_idle_timeout() -> u64 { 300 } +pub const fn default_hls_segment_repair_apply_to_first_segments() -> u8 { 1 } +pub const fn default_hls_segment_repair_max_parallel_repairs() -> usize { 1 } +pub const fn default_hls_segment_repair_low_size_increase_percent() -> u8 { 2 } +pub const fn default_hls_segment_repair_medium_size_increase_percent() -> u8 { 5 } +pub const fn default_hls_segment_repair_high_size_increase_percent() -> u8 { 20 } +pub const fn default_hls_segment_repair_postprocess_timeout_ms() -> u64 { 2_000 } +pub const fn default_hls_corrupt_segment_watchdog_max_parallel_jobs() -> usize { 1 } + +// HLS manifest / fallback filename constants. +pub const HLS_EXT: &str = ".m3u8"; +pub const DASH_EXT: &str = ".mpd"; +pub const HLS_PREFIX: &str = "hls"; +pub const CUSTOM_VIDEO_PREFIX: &str = "cvs"; +pub const HLS_EXT_QUERY: &str = ".m3u8?"; +pub const HLS_EXT_FRAGMENT: &str = ".m3u8#"; +pub const DASH_EXT_QUERY: &str = ".mpd?"; +pub const DASH_EXT_FRAGMENT: &str = ".mpd#"; +pub const CHANNEL_UNAVAILABLE: &str = "channel_unavailable.ts"; +pub const USER_CONNECTIONS_EXHAUSTED: &str = "user_connections_exhausted.ts"; +pub const PROVIDER_CONNECTIONS_EXHAUSTED: &str = "provider_connections_exhausted.ts"; +pub const LOW_PRIORITY_PREEMPTED: &str = "low_priority_preempted.ts"; +pub const USER_ACCOUNT_EXPIRED: &str = "user_account_expired.ts"; +pub const PANEL_API_PROVISIONING: &str = "panel_api_provisioning.ts"; +pub const HLS_SESSION_OR_LEASE_EXPIRED: &str = "hls_session_or_lease_expired.ts"; +pub const PANEL_API_PROVISIONING_HLS_SEGMENT_COUNT: usize = 6; +pub const PANEL_API_PROVISIONING_HLS_SEGMENT_PREFIX: &str = "panel_api_provisioning_hls_"; diff --git a/shared/src/defaults/library.rs b/shared/src/defaults/library.rs new file mode 100644 index 000000000..009274593 --- /dev/null +++ b/shared/src/defaults/library.rs @@ -0,0 +1,46 @@ +//! Library defaults: extensions, categories, thumbnails, processing-order +//! predicate, video-DTO empty predicate, config-target-options predicate. + +use crate::model::{LibraryMetadataFormat, ProcessingOrder, VideoConfigDto}; + +pub const DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS: &[&str] = &["mp4", "mkv", "avi", "mov", "ts", "m4v", "webm"]; + +pub fn default_supported_library_extensions() -> Vec { + DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS.iter().map(|s| (*s).to_owned()).collect() +} + +pub fn is_default_supported_library_extensions(v: &[String]) -> bool { + v.len() == DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS.len() + && v.iter().zip(DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS).all(|(a, b)| a == b) +} + +pub const DEFAULT_VIDEO_EXTENSIONS: &[&str] = &["mkv", "avi", "mp4", "mpeg", "divx", "mov"]; + +pub fn default_supported_video_extensions() -> Vec { + DEFAULT_VIDEO_EXTENSIONS.iter().map(|s| (*s).to_owned()).collect() +} + +pub fn is_default_supported_video_extensions(v: &[String]) -> bool { + v.len() == DEFAULT_VIDEO_EXTENSIONS.len() && v.iter().zip(DEFAULT_VIDEO_EXTENSIONS).all(|(a, b)| a == b) +} + +pub fn default_storage_formats() -> Vec { vec![] } +pub fn default_movie_category() -> String { String::from("Local Movies") } +pub fn default_series_category() -> String { String::from("Local TV Shows") } + +pub fn default_thumbnail_width() -> u32 { 320 } +pub fn default_thumbnail_height() -> u32 { 180 } +pub fn default_thumbnail_quality() -> u8 { 75 } + +pub fn is_default_processing_order(p: &ProcessingOrder) -> bool { *p == ProcessingOrder::default() } + +pub const fn default_probe_live_interval() -> u32 { 120 } +pub const fn is_default_probe_live_interval(v: &u32) -> bool { *v == default_probe_live_interval() } + +pub fn is_none_or_empty_video(video: &Option) -> bool { + video.as_ref().is_none_or(VideoConfigDto::is_empty) +} + +pub fn is_config_target_options_empty(v: &Option) -> bool { + v.as_ref().is_none_or(|c| c.is_empty()) +} diff --git a/shared/src/defaults/media_server.rs b/shared/src/defaults/media_server.rs new file mode 100644 index 000000000..001fe8ce9 --- /dev/null +++ b/shared/src/defaults/media_server.rs @@ -0,0 +1,6 @@ +//! Media-server catalog defaults. + +default_eq_fns!( + default_media_server_catalog_page_size, is_default_media_server_catalog_page_size, u16, 100; + default_media_server_catalog_request_delay_ms, is_default_media_server_catalog_request_delay_ms, u64, 250; +); diff --git a/shared/src/defaults/metadata.rs b/shared/src/defaults/metadata.rs new file mode 100644 index 000000000..f11b4db52 --- /dev/null +++ b/shared/src/defaults/metadata.rs @@ -0,0 +1,59 @@ +//! Metadata-update defaults: paths, queue/log intervals, probe/resolve backoff, +//! queue/backoff/attempt knobs, ffprobe sizes & durations. + +use crate::model::ByteSize; + +pub const DEFAULT_METADATA_PATH: &str = "metadata"; +pub fn default_metadata_path() -> String { DEFAULT_METADATA_PATH.to_string() } +pub fn is_default_metadata_path(s: &str) -> bool { s == DEFAULT_METADATA_PATH } + +// All queue/log/cooldown/retry-duration defaults are human-readable strings +// (e.g. "30s", "1h", "7d"); the macro emits a non-allocating comparison via +// the cached `&'static str` returned by the default arm. +default_eq_fns!( + default_metadata_queue_log_interval, is_default_metadata_queue_log_interval, str, "30s"; + default_metadata_progress_log_interval, is_default_metadata_progress_log_interval, str, "15s"; + default_metadata_max_resolve_retry_backoff, is_default_metadata_max_resolve_retry_backoff, str, "1h"; + default_metadata_resolve_min_retry_base, is_default_metadata_resolve_min_retry_base, str, "5s"; + default_metadata_resolve_exhaustion_reset_gap, is_default_metadata_resolve_exhaustion_reset_gap, str, "1h"; + default_metadata_probe_cooldown, is_default_metadata_probe_cooldown, str, "7d"; + default_metadata_retry_delay, is_default_metadata_retry_delay, str, "2s"; + default_metadata_probe_retry_load_retry_delay, is_default_metadata_probe_retry_load_retry_delay, str, "1m"; + default_metadata_worker_idle_timeout, is_default_metadata_worker_idle_timeout, str, "1m"; + default_metadata_probe_retry_backoff_step_1, is_default_metadata_probe_retry_backoff_step_1, str, "10m"; + default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_2, str, "30m"; + default_metadata_probe_retry_backoff_step_3, is_default_metadata_probe_retry_backoff_step_3, str, "1h"; +); + +default_eq_fns!( + default_metadata_max_attempts_resolve, is_default_metadata_max_attempts_resolve, u8, 3; + default_metadata_max_attempts_probe, is_default_metadata_max_attempts_probe, u8, 3; + default_metadata_backoff_jitter_percent, is_default_metadata_backoff_jitter_percent, u8, 20; + default_metadata_max_queue_size, is_default_metadata_max_queue_size, usize, 100_000; + default_metadata_no_change_cache_ttl_secs, is_default_metadata_no_change_cache_ttl_secs, u64, 3600; + default_metadata_probe_fairness_resolve_burst, is_default_metadata_probe_fairness_resolve_burst, usize, 200; +); + +default_eq_fns!( + default_metadata_ffprobe_analyze_duration, is_default_metadata_ffprobe_analyze_duration, str, "10s"; + default_metadata_ffprobe_live_analyze_duration, is_default_metadata_ffprobe_live_analyze_duration, str, "5s"; +); + +// `ByteSize` defaults — non-numeric, non-`str`, kept as manual impls. Keep +// them minimal; if a third `ByteSize` default appears, extend the macro with +// a `byte_size` arm instead of repeating this pattern. +pub fn default_metadata_ffprobe_probe_size() -> ByteSize { ByteSize::new("10MB") } +pub fn is_default_metadata_ffprobe_probe_size(v: &ByteSize) -> bool { v == &default_metadata_ffprobe_probe_size() } +pub fn default_metadata_ffprobe_live_probe_size() -> ByteSize { ByteSize::new("5MB") } +pub fn is_default_metadata_ffprobe_live_probe_size(v: &ByteSize) -> bool { + v == &default_metadata_ffprobe_live_probe_size() +} + +pub const fn default_probe_user_priority() -> i8 { 127 } +pub const fn is_default_probe_user_priority(v: &i8) -> bool { *v == default_probe_user_priority() } +pub const fn default_user_priority() -> i8 { 0 } +pub const fn is_default_user_priority(v: &i8) -> bool { *v == default_user_priority() } + +pub fn is_none_or_empty_metadata_update(metadata_update: &Option) -> bool { + metadata_update.as_ref().is_none_or(crate::model::MetadataUpdateConfigDto::is_empty) +} diff --git a/shared/src/defaults/mod.rs b/shared/src/defaults/mod.rs new file mode 100644 index 000000000..0ba969981 --- /dev/null +++ b/shared/src/defaults/mod.rs @@ -0,0 +1,111 @@ +//! Default-value helpers grouped by domain. +//! +//! Items are re-exported at the crate root via `shared::utils::*` (see +//! `shared/src/utils/mod.rs`) so all existing `use shared::utils::default_xxx;` +//! imports keep working unchanged. +//! +//! The 14 leaf files in this directory are flat by external-service name +//! (auth, epg, hdhomerun, hls, library, media_server, metadata, monitoring, +//! network, paths, primitives, stream_history, tmdb, trakt). To answer +//! "what is the default for HLS?" in one grep, the same items are also +//! re-exported under three logical groups: +//! +//! - [`config`] — streaming-side defaults: auth, hls, hdhomerun, library, media_server +//! - [`integrations`] — external-service defaults: epg, metadata, tmdb, trakt +//! - [`runtime`] — runtime/infrastructure defaults: monitoring, network, paths, primitives, stream_history +//! +//! New defaults can be added to either the flat layout or a group module; +//! both are re-exported at the crate root. + +/// Generates `default_*` / `is_default_*` pairs for value types. +/// +/// Numeric arms (`u8`/`u16`/`u32`/`usize`/`i64`/`u64`) emit `const fn` returning +/// the value directly. The `str` arm emits `String`-returning defaults so +/// existing call sites that store into `String`-typed config fields keep +/// working unchanged (one heap allocation per `default_*` invocation; +/// defaults are read at config-load time, not in a hot path). +/// +/// Add a new arm (e.g. `byte_size`) when a non-numeric type starts repeating +/// the same `default_*` + `is_default_*` shape. +#[macro_export] +macro_rules! default_eq_fns { + // String-typed defaults (e.g. duration strings "30s", "1h", "7d"). + // This arm MUST come first — the numeric arm below would otherwise match + // `str, "30s"` and emit `pub const fn ...() -> str { ... }` (str is + // unsized, so the generated code fails to compile). + ($( $default_fn:ident, $is_default_fn:ident, str, $value:literal; )* ) => { + $( + pub fn $default_fn() -> String { $value.to_string() } + pub fn $is_default_fn(v: &String) -> bool { v == &$default_fn() } + )* + }; + // Numeric: literal default folds into a const. + ($( $default_fn:ident, $is_default_fn:ident, $ty:ty, $value:expr; )* ) => { + $( + pub const fn $default_fn() -> $ty { $value } + pub const fn $is_default_fn(v: &$ty) -> bool { *v == $default_fn() } + )* + }; +} + +/// Generates `Display` + `FromStr` for a snake_case-renamed string enum. +/// String labels must equal the `#[serde(rename_all = "snake_case")]` form so +/// the wire format, `Display`, and `FromStr` agree on a single canonical name. +#[macro_export] +macro_rules! impl_str_enum { + ( $enum:ty, $err_label:expr, $( $variant:ident => $str:literal ),+ $(,)? ) => { + impl std::fmt::Display for $enum { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + $( Self::$variant => f.write_str($str), )+ + } + } + } + impl std::str::FromStr for $enum { + type Err = String; + fn from_str(value: &str) -> Result { + match value { + $( $str => Ok(Self::$variant), )+ + _ => Err(format!("Unknown {}: {value}", $err_label)), + } + } + } + }; +} + +mod auth; +mod epg; +mod hdhomerun; +mod hls; +mod library; +mod media_server; +mod metadata; +mod monitoring; +mod network; +mod paths; +mod primitives; +mod stream_history; +mod tmdb; +mod trakt; + +pub use self::{ + auth::*, epg::*, hdhomerun::*, hls::*, library::*, media_server::*, metadata::*, monitoring::*, network::*, + paths::*, primitives::*, stream_history::*, tmdb::*, trakt::*, +}; + +/// Streaming-side defaults (auth, hls, hdhomerun, library, media_server). +/// Re-exports the flat items under a logical group so callers asking +/// "what is the default for HLS?" can grep one place. +pub mod config { + pub use super::{auth::*, hdhomerun::*, hls::*, library::*, media_server::*}; +} + +/// External-service defaults (epg, metadata, tmdb, trakt). +pub mod integrations { + pub use super::{epg::*, metadata::*, tmdb::*, trakt::*}; +} + +/// Runtime/infrastructure defaults (monitoring, network, paths, primitives, stream_history). +pub mod runtime { + pub use super::{monitoring::*, network::*, paths::*, primitives::*, stream_history::*}; +} diff --git a/shared/src/defaults/monitoring.rs b/shared/src/defaults/monitoring.rs new file mode 100644 index 000000000..383dc1d2c --- /dev/null +++ b/shared/src/defaults/monitoring.rs @@ -0,0 +1,5 @@ +//! Resource-monitoring defaults (warn/critical percentages, repeat interval). + +pub fn default_warn_percent() -> f64 { 80.0 } +pub fn default_critical_percent() -> f64 { 95.0 } +pub fn default_repeat_interval_secs() -> u64 { 3600 } diff --git a/shared/src/defaults/network.rs b/shared/src/defaults/network.rs new file mode 100644 index 000000000..836d29313 --- /dev/null +++ b/shared/src/defaults/network.rs @@ -0,0 +1,30 @@ +//! Generic networking defaults: resolve/probe delays, grace periods, retry, +//! interner GC, port, user-agent, background flags. + +default_eq_fns!( + default_resolve_delay_secs, is_default_resolve_delay_secs, u16, 2; + default_probe_delay_secs, is_default_probe_delay_secs, u16, 2; + default_grace_period_millis, is_default_grace_period_millis, u64, 2000; + default_shared_burst_buffer_mb, is_default_shared_burst_buffer_mb, u64, 12; + default_grace_period_timeout_secs, is_default_grace_period_timeout_secs, u64, 4; + default_catchup_session_ttl_secs, is_default_catchup_session_ttl_secs, u64, 45; + default_connect_timeout_secs, is_default_connect_timeout_secs, u32, 6; + default_resource_retry_attempts, is_default_resource_retry_attempts, u32, 3; + default_resource_retry_backoff_ms, is_default_resource_retry_backoff_ms, u64, 250; + default_interner_gc_interval_secs, is_default_interner_gc_interval_secs, u32, 180; + default_interner_gc_min_pool_size, is_default_interner_gc_min_pool_size, u32, 100; + default_custom_stream_response_error_status, is_default_custom_stream_response_error_status, u16, 502; +); + +pub const fn default_resource_retry_backoff_multiplier() -> f64 { 1.0 } +pub const F64_DEFAULT_EPSILON: f64 = 1e-9; +pub const fn is_default_resource_retry_backoff_multiplier(v: &f64) -> bool { + (*v - default_resource_retry_backoff_multiplier()).abs() < F64_DEFAULT_EPSILON +} + +pub const fn default_resolve_background() -> bool { true } +pub const fn default_xtream_live_stream_use_prefix() -> bool { true } + +pub const DEFAULT_PORT: u16 = 8901; +pub const DEFAULT_USER_AGENT: &str = "VLC/3.0.16 LibVLC/3.0.16"; +pub fn default_default_user_agent() -> Option { Some(DEFAULT_USER_AGENT.to_string()) } diff --git a/shared/src/defaults/paths.rs b/shared/src/defaults/paths.rs new file mode 100644 index 000000000..d1b54d183 --- /dev/null +++ b/shared/src/defaults/paths.rs @@ -0,0 +1,118 @@ +//! Path defaults and path-blank-or-default predicates. + +// Filesystem constants used as defaults for config paths. +pub const CONFIG_PATH: &str = "config"; +pub const CONFIG_FILE: &str = "config.yml"; +pub const SOURCE_FILE: &str = "source.yml"; +pub const MAPPING_FILE: &str = "mapping.yml"; +pub const TEMPLATE_FILE: &str = "template.yml"; +pub const API_PROXY_FILE: &str = "api-proxy.yml"; +pub const USER_FILE: &str = "user.txt"; +pub const USER_GROUP_FILE: &str = "groups.txt"; + +pub const DEFAULT_WEB_DIR: &str = "web"; +pub const DEFAULT_BACKUP_DIR: &str = "backup"; +pub const DEFAULT_CACHE_DIR: &str = "cache"; +pub const DEFAULT_STORAGE_TEMP_DIR: &str = "tmp"; +pub const DEFAULT_USER_CONFIG_DIR: &str = "user_config"; +pub const DEFAULT_DOWNLOAD_DIR: &str = "downloads"; +pub const DEFAULT_STORAGE_DIR: &str = "data"; // TODO rename to storage and use data for config, storage, ... +pub const DEFAULT_CUSTOM_STREAM_RESPONSE_PATH: &str = "resources"; + +pub fn get_default_web_root() -> String { DEFAULT_WEB_DIR.to_string() } +pub fn is_blank_or_default_web_root(value: &str) -> bool { + let normalized = value.trim().replace('\\', "/"); + if normalized.is_empty() { + return true; + } + + let normalized = normalized.trim_end_matches('/'); + normalized.trim_start_matches("./") == DEFAULT_WEB_DIR +} + +pub fn is_default_dir_path(value: &str, default_dir: &str) -> bool { + let normalized = value.trim().replace('\\', "/"); + let normalized = normalized.trim_end_matches('/'); + let normalized = normalized.trim_start_matches("./"); + let default_dir = default_dir.trim().replace('\\', "/"); + let default_dir = default_dir.trim_end_matches('/'); + normalized == default_dir +} + +pub fn is_blank_or_default_download_dir(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_DOWNLOAD_DIR)) +} +pub fn default_download_dir() -> Option { Some(DEFAULT_DOWNLOAD_DIR.to_string()) } + +pub fn default_main_storage_dir() -> Option { Some(DEFAULT_STORAGE_DIR.to_string()) } +pub fn default_main_backup_dir() -> Option { Some(DEFAULT_BACKUP_DIR.to_string()) } +pub fn default_main_user_config_dir() -> Option { Some(DEFAULT_USER_CONFIG_DIR.to_string()) } +pub fn default_main_mapping_path() -> Option { Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}")) } +pub fn default_main_template_path() -> Option { Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}")) } +pub fn default_custom_stream_response_path() -> Option { Some(DEFAULT_CUSTOM_STREAM_RESPONSE_PATH.to_string()) } +pub fn default_user_file_path() -> Option { Some(format!("./{CONFIG_PATH}/{USER_FILE}")) } +pub fn default_user_group_file_path() -> Option { Some(format!("./{CONFIG_PATH}/{USER_GROUP_FILE}")) } + +fn is_default_config_file_path(value: &str, file_name: &str) -> bool { + let normalized = value.trim().replace('\\', "/"); + let normalized = normalized.trim_start_matches("./"); + normalized == file_name + || normalized.rsplit_once('/').is_some_and(|(dir, file)| dir == CONFIG_PATH && file == file_name) +} + +pub fn is_blank_or_default_custom_stream_response_path(path: &Option) -> bool { + path.as_ref() + .is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH)) +} + +pub fn is_blank_or_default_mapping_path(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, MAPPING_FILE)) +} + +pub fn is_blank_or_default_template_path(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, TEMPLATE_FILE)) +} + +pub fn is_blank_or_default_storage_dir(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_STORAGE_DIR)) +} + +pub fn is_blank_or_default_backup_dir(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_BACKUP_DIR)) +} + +pub fn is_blank_or_default_user_config_dir(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_USER_CONFIG_DIR)) +} + +pub fn is_blank_or_default_user_file_path(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, USER_FILE)) +} + +pub fn is_blank_or_default_user_group_file_path(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, USER_GROUP_FILE)) +} + +pub fn is_blank_or_default_cache_dir(path: &Option) -> bool { + path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_CACHE_DIR)) +} + +pub fn normalize_optional_dir(path: &Option, default_dir: &str) -> Option { + path.as_ref().and_then(|value| { + if value.trim().is_empty() || is_default_dir_path(value, default_dir) { + None + } else { + Some(value.clone()) + } + }) +} + +pub fn normalize_optional_config_file_path(path: &Option, default_file_name: &str) -> Option { + path.as_ref().and_then(|value| { + if value.trim().is_empty() || is_default_config_file_path(value, default_file_name) { + None + } else { + Some(value.clone()) + } + }) +} diff --git a/shared/src/defaults/primitives.rs b/shared/src/defaults/primitives.rs new file mode 100644 index 000000000..a0a98a316 --- /dev/null +++ b/shared/src/defaults/primitives.rs @@ -0,0 +1,24 @@ +//! Generic predicates and tiny defaults shared across all config DTOs. + +use crate::model::{ClusterFlags, RuntimeConfigReportFormat}; + +pub const fn is_zero_u16(v: &u16) -> bool { *v == 0 } +pub const fn is_zero_i16(v: &i16) -> bool { *v == 0 } +pub const fn is_zero_u32(v: &u32) -> bool { *v == 0 } +pub const fn is_true(v: &bool) -> bool { *v } +pub const fn is_false(v: &bool) -> bool { !*v } +pub const fn default_as_true() -> bool { true } + +pub fn is_empty_optional_vec(s: &Option>) -> bool { s.as_ref().is_none_or(|v| v.is_empty()) } + +pub fn default_as_default() -> String { "default".into() } + +pub fn default_page() -> u32 { 1 } + +pub fn default_page_size() -> u16 { 25 } + +pub const fn is_default_runtime_config_report_format(value: &RuntimeConfigReportFormat) -> bool { + matches!(value, RuntimeConfigReportFormat::Yaml) +} + +pub fn is_cluster_optional(cf: &Option) -> bool { cf.is_none_or(|c| c.is_all()) } diff --git a/shared/src/defaults/stream_history.rs b/shared/src/defaults/stream_history.rs new file mode 100644 index 000000000..5bd4ae142 --- /dev/null +++ b/shared/src/defaults/stream_history.rs @@ -0,0 +1,17 @@ +//! Stream-history defaults. + +pub const DEFAULT_STREAM_HISTORY_BATCH_SIZE: usize = 128; +pub const DEFAULT_STREAM_HISTORY_RETENTION_DAYS: u16 = 30; +pub const DEFAULT_STREAM_HISTORY_DIR: &str = "stream_history"; + +pub fn default_stream_history_batch_size() -> usize { DEFAULT_STREAM_HISTORY_BATCH_SIZE } +pub fn default_stream_history_retention_days() -> u16 { DEFAULT_STREAM_HISTORY_RETENTION_DAYS } +pub fn default_stream_history_directory() -> String { DEFAULT_STREAM_HISTORY_DIR.to_string() } + +pub fn is_default_stream_history_batch_size(batch_size: &usize) -> bool { + *batch_size == DEFAULT_STREAM_HISTORY_BATCH_SIZE +} +pub fn is_default_stream_history_retention_days(retention_days: &u16) -> bool { + *retention_days == DEFAULT_STREAM_HISTORY_RETENTION_DAYS +} +pub fn is_blank_stream_history_directory(directory: &str) -> bool { directory.trim().is_empty() } diff --git a/shared/src/defaults/tmdb.rs b/shared/src/defaults/tmdb.rs new file mode 100644 index 000000000..016923627 --- /dev/null +++ b/shared/src/defaults/tmdb.rs @@ -0,0 +1,23 @@ +//! TMDB defaults: API key, rate limit, language, cache duration, cooldown. + +pub const TMDB_API_KEY: &str = "4219e299c89411838049ab0dab19ebd5"; + +pub fn default_tmdb_api_key() -> Option { Some(TMDB_API_KEY.to_string()) } +pub fn is_tmdb_default_api_key(s: &Option) -> bool { s.as_ref().is_none_or(|s| s == TMDB_API_KEY) } + +pub const DEFAULT_TMDB_RATE_LIMIT_MS: u64 = 250; +pub const DEFAULT_TMDB_CACHE_DURATION_DAYS: u32 = 30; +pub const DEFAULT_TMDB_LANGUAGE: &str = "en-US"; + +pub const fn default_tmdb_rate_limit_ms() -> u64 { DEFAULT_TMDB_RATE_LIMIT_MS } +pub const fn default_tmdb_cache_duration_days() -> u32 { DEFAULT_TMDB_CACHE_DURATION_DAYS } +pub fn default_tmdb_language() -> String { DEFAULT_TMDB_LANGUAGE.to_owned() } +pub const fn is_default_tmdb_rate_limit_ms(v: &u64) -> bool { *v == DEFAULT_TMDB_RATE_LIMIT_MS } +pub const fn is_default_tmdb_cache_duration_days(v: &u32) -> bool { *v == DEFAULT_TMDB_CACHE_DURATION_DAYS } +pub fn is_default_tmdb_language(v: &String) -> bool { v == DEFAULT_TMDB_LANGUAGE } + +pub const fn default_tmdb_match_threshold() -> u16 { 86 } +pub const fn is_default_tmdb_match_threshold(v: &u16) -> bool { *v == default_tmdb_match_threshold() } + +pub fn default_metadata_tmdb_cooldown() -> String { "7d".to_string() } +pub fn is_default_metadata_tmdb_cooldown(v: &String) -> bool { *v == default_metadata_tmdb_cooldown() } diff --git a/shared/src/defaults/trakt.rs b/shared/src/defaults/trakt.rs new file mode 100644 index 000000000..ed5d28a1c --- /dev/null +++ b/shared/src/defaults/trakt.rs @@ -0,0 +1,10 @@ +//! Trakt defaults. + +pub const TRAKT_API_KEY: &str = "0183a05ad97098d87287fe46da4ae286f434f32e8e951caad4cc147c947d79a3"; +pub const TRAKT_API_VERSION: &str = "2"; +pub const TRAKT_API_URL: &str = "https://api.trakt.tv"; + +pub fn default_trakt_api_key() -> String { String::from(TRAKT_API_KEY) } +pub fn default_trakt_api_version() -> String { String::from(TRAKT_API_VERSION) } +pub fn default_trakt_api_url() -> String { String::from(TRAKT_API_URL) } +pub fn default_trakt_fuzzy_threshold() -> u8 { 80 } diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 2e3993ada..ce4691f04 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -1,3 +1,4 @@ +pub mod defaults; pub mod error; pub mod foundation; pub mod model; diff --git a/shared/src/model/config/api.rs b/shared/src/model/config/api.rs index 9e591d26f..f6f48cba6 100644 --- a/shared/src/model/config/api.rs +++ b/shared/src/model/config/api.rs @@ -1,4 +1,4 @@ -use crate::utils::{get_default_web_root, is_blank_or_default_web_root}; +use crate::defaults::{get_default_web_root, is_blank_or_default_web_root}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] #[serde(deny_unknown_fields)] diff --git a/shared/src/model/config/api_proxy.rs b/shared/src/model/config/api_proxy.rs index 54cbf836a..2b9c01630 100644 --- a/shared/src/model/config/api_proxy.rs +++ b/shared/src/model/config/api_proxy.rs @@ -1,7 +1,8 @@ use crate::{ + defaults::{default_auth_error_status, is_default_auth_error_status, is_false}, error::TuliproxError, model::ProxyUserCredentialsDto, - utils::{default_auth_error_status, is_blank_optional_string, is_default_auth_error_status, is_false}, + utils::is_blank_optional_string, }; use std::collections::HashSet; diff --git a/shared/src/model/config/api_user.rs b/shared/src/model/config/api_user.rs index 90ac456d9..ec50b0196 100644 --- a/shared/src/model/config/api_user.rs +++ b/shared/src/model/config/api_user.rs @@ -1,10 +1,8 @@ use crate::{ + defaults::{default_as_true, default_user_priority, is_cluster_optional, is_default_user_priority, is_true}, error::TuliproxError, model::{ClusterFlags, NetworkAccessDto, ProxyType, ProxyUserStatus, XtreamCluster}, - utils::{ - default_as_true, default_user_priority, deserialize_timestamp, is_blank_optional_string, is_cluster_optional, - is_default_user_priority, is_true, - }, + utils::{deserialize_timestamp, is_blank_optional_string}, }; #[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] diff --git a/shared/src/model/config/base.rs b/shared/src/model/config/base.rs index b608130ba..5cee4123b 100644 --- a/shared/src/model/config/base.rs +++ b/shared/src/model/config/base.rs @@ -1,16 +1,10 @@ use crate::{ - error::TuliproxError, - model::{ - ConfigApiDto, HdHomeRunConfigDto, IpCheckConfigDto, LibraryConfigDto, LogConfigDto, MessagingConfigDto, - MetadataUpdateConfigDto, ProxyConfigDto, ReverseProxyConfigDto, ScheduleConfigDto, VideoConfigDto, - WebUiConfigDto, - }, - utils::{ + defaults::{ default_as_true, default_connect_timeout_secs, default_custom_stream_response_error_status, default_custom_stream_response_path, default_default_user_agent, default_interner_gc_interval_secs, default_interner_gc_min_pool_size, default_main_backup_dir, default_main_mapping_path, default_main_storage_dir, default_main_template_path, default_main_user_config_dir, - default_supported_video_extensions, is_blank_optional_string, is_blank_or_default_backup_dir, + default_supported_video_extensions, is_blank_or_default_backup_dir, is_blank_or_default_custom_stream_response_path, is_blank_or_default_mapping_path, is_blank_or_default_storage_dir, is_blank_or_default_template_path, is_blank_or_default_user_config_dir, is_default_connect_timeout_secs, is_default_custom_stream_response_error_status, @@ -19,6 +13,13 @@ use crate::{ normalize_optional_config_file_path, normalize_optional_dir, DEFAULT_BACKUP_DIR, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH, DEFAULT_STORAGE_DIR, DEFAULT_USER_CONFIG_DIR, MAPPING_FILE, TEMPLATE_FILE, }, + error::TuliproxError, + model::{ + ConfigApiDto, HdHomeRunConfigDto, IpCheckConfigDto, LibraryConfigDto, LogConfigDto, MessagingConfigDto, + MetadataUpdateConfigDto, ProxyConfigDto, ReverseProxyConfigDto, ScheduleConfigDto, VideoConfigDto, + WebUiConfigDto, + }, + utils::is_blank_optional_string, }; #[allow(clippy::struct_excessive_bools)] @@ -458,7 +459,7 @@ impl ConfigDto { #[cfg(test)] mod tests { use super::*; - use crate::utils::{default_supported_video_extensions, CONFIG_PATH}; + use crate::defaults::{default_supported_video_extensions, CONFIG_PATH}; use serde_json::json; #[test] diff --git a/shared/src/model/config/byte_size.rs b/shared/src/model/config/byte_size.rs new file mode 100644 index 000000000..bcf5d0eff --- /dev/null +++ b/shared/src/model/config/byte_size.rs @@ -0,0 +1,94 @@ +use crate::utils::parse_size_base_2; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ByteSize(String); + +impl ByteSize { + pub fn new(value: impl Into) -> Self { Self(value.into()) } + + pub fn as_str(&self) -> &str { self.0.as_str() } + + pub fn clean_or_default(&mut self, default_value: &str) { + let trimmed = self.0.trim(); + self.0 = if trimmed.is_empty() { default_value.to_string() } else { trimmed.to_string() }; + } + + pub fn parse_bytes(&self) -> Result { parse_size_base_2(self.0.as_str()) } +} + +impl From for ByteSize { + fn from(value: String) -> Self { Self::new(value) } +} + +impl From<&str> for ByteSize { + fn from(value: &str) -> Self { Self::new(value) } +} + +impl std::fmt::Display for ByteSize { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } +} + +impl Serialize for ByteSize { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.0.as_str()) + } +} + +impl<'de> Deserialize<'de> for ByteSize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct ByteSizeVisitor; + + impl<'de> de::Visitor<'de> for ByteSizeVisitor { + type Value = ByteSize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a byte size string or unsigned integer") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(ByteSize(value.to_string())) + } + + fn visit_string(self, value: String) -> Result + where + E: de::Error, + { + Ok(ByteSize(value)) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + Ok(ByteSize(value.to_string())) + } + } + + deserializer.deserialize_any(ByteSizeVisitor) + } +} + +#[cfg(test)] +mod tests { + use super::ByteSize; + + #[test] + fn parse_bytes_handles_binary_units() { + // parse_bytes is binary (parse_size_base_2): 10GB = 10 * 1024^3, 512MB = 512 * 1024^2. + assert_eq!(ByteSize::new("10GB").parse_bytes().expect("10GB should parse"), 10_737_418_240); + assert_eq!(ByteSize::new("512MB").parse_bytes().expect("512MB should parse"), 536_870_912); + assert_eq!(ByteSize::new("1048576").parse_bytes().expect("bytes should parse"), 1_048_576); + assert_eq!(ByteSize::new("1GiB").parse_bytes().expect("1GiB should parse"), 1_073_741_824); + assert_eq!(ByteSize::new("0").parse_bytes().expect("0 should parse"), 0); + } +} diff --git a/shared/src/model/config/cache.rs b/shared/src/model/config/cache.rs index 77018d0fd..da8248a56 100644 --- a/shared/src/model/config/cache.rs +++ b/shared/src/model/config/cache.rs @@ -1,25 +1,28 @@ use crate::{ + defaults::{is_blank_or_default_cache_dir, DEFAULT_CACHE_DIR}, error::TuliproxError, - utils::{ - is_blank_optional_str, is_blank_optional_string, is_blank_or_default_cache_dir, parse_size_base_2, - DEFAULT_CACHE_DIR, - }, + model::ByteSize, + utils::is_blank_optional_str, }; +fn is_blank_optional_byte_size(value: &Option) -> bool { + value.as_ref().is_none_or(|v| v.as_str().trim().is_empty()) +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] #[serde(deny_unknown_fields)] pub struct CacheConfigDto { #[serde(default)] pub enabled: bool, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub size: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_byte_size")] + pub size: Option, #[serde(default, alias = "dir", skip_serializing_if = "is_blank_or_default_cache_dir")] pub directory: Option, } impl CacheConfigDto { pub fn is_empty(&self) -> bool { - !self.enabled && is_blank_optional_str(self.size.as_deref()) && is_blank_optional_str(self.directory.as_deref()) + !self.enabled && is_blank_optional_byte_size(&self.size) && is_blank_optional_str(self.directory.as_deref()) } pub(crate) fn prepare(&mut self, _storage_dir: &str) -> Result<(), TuliproxError> { @@ -31,9 +34,8 @@ impl CacheConfigDto { } if let Some(val) = self.size.as_ref() { - if let Err(err) = parse_size_base_2(val) { - return Err(TuliproxError::ConfigCache(format!("Failed to read cache size: {err}"))); - } + val.parse_bytes() + .map_err(|err| TuliproxError::ConfigCache(format!("Failed to read cache size: {err}")))?; } } Ok(()) diff --git a/shared/src/model/config/epg.rs b/shared/src/model/config/epg.rs index 7615ba81a..8ce929f27 100644 --- a/shared/src/model/config/epg.rs +++ b/shared/src/model/config/epg.rs @@ -1,4 +1,4 @@ -use crate::{error::TuliproxError, model::EpgSmartMatchConfigDto, utils::is_false}; +use crate::{defaults::is_false, error::TuliproxError, model::EpgSmartMatchConfigDto}; const AUTO_URL: &str = "auto"; diff --git a/shared/src/model/config/epg_smart_match.rs b/shared/src/model/config/epg_smart_match.rs index e2e138f3c..b6da48979 100644 --- a/shared/src/model/config/epg_smart_match.rs +++ b/shared/src/model/config/epg_smart_match.rs @@ -1,11 +1,11 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_epg_best_match_threshold, default_epg_match_threshold, default_epg_name_prefix_separator, default_epg_normalize_regex, default_epg_strip, is_default_epg_best_match_threshold, is_default_epg_match_threshold, is_default_epg_name_prefix_separator, is_default_epg_normalize_regex, is_default_epg_strip, is_false, }, + error::TuliproxError, }; use log::warn; use std::fmt::Display; diff --git a/shared/src/model/config/hdhomerun.rs b/shared/src/model/config/hdhomerun.rs index 219de5418..87b43a18c 100644 --- a/shared/src/model/config/hdhomerun.rs +++ b/shared/src/model/config/hdhomerun.rs @@ -1,12 +1,12 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_as_true, default_device_type, default_device_udn, default_firmware_name, default_firmware_version, - default_friendly_name, default_manufacturer, default_model_name, generate_hdhr_device_id, - generate_hdhr_device_id_from_base, hash_string, is_default_device_type, is_default_device_udn, + default_friendly_name, default_manufacturer, default_model_name, is_default_device_type, is_default_device_udn, is_default_firmware_name, is_default_firmware_version, is_default_friendly_name, is_default_manufacturer, - is_default_model_name, is_false, is_true, validate_hdhr_device_id, + is_default_model_name, is_false, is_true, }, + error::TuliproxError, + utils::{generate_hdhr_device_id, generate_hdhr_device_id_from_base, hash_string, validate_hdhr_device_id}, }; use log::warn; use std::collections::HashSet; diff --git a/shared/src/model/config/hls.rs b/shared/src/model/config/hls.rs new file mode 100644 index 000000000..4d9808d14 --- /dev/null +++ b/shared/src/model/config/hls.rs @@ -0,0 +1,602 @@ +use crate::{ + defaults::{ + default_hls_cache_bytes, default_hls_cache_bytes_per_session, default_hls_cache_duration, + default_hls_corrupt_segment_watchdog_max_parallel_jobs, default_hls_max_concurrent_segment_fetches_global, + default_hls_max_concurrent_segment_fetches_per_session, default_hls_max_segments_prefetch, + default_hls_origin_manifest_timeout_ms, default_hls_origin_segment_timeout_ms, + default_hls_segment_repair_apply_to_first_segments, default_hls_segment_repair_high_size_increase_percent, + default_hls_segment_repair_low_size_increase_percent, default_hls_segment_repair_max_parallel_repairs, + default_hls_segment_repair_medium_size_increase_percent, default_hls_segment_repair_postprocess_timeout_ms, + default_hls_session_idle_timeout, DEFAULT_HLS_CACHE_BYTES, DEFAULT_HLS_CACHE_BYTES_PER_SESSION, + }, + error::TuliproxError, + model::{ + ByteSize, HlsCorruptSegmentWatchdogMode, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, + }, + utils::is_blank_optional_string, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsStripConfigDto { + #[serde(default)] + pub mode: HlsStripMode, + #[serde(default)] + pub value: u64, +} + +impl HlsStripConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsManifestRecoveryBurstConfigDto { + #[serde(default)] + pub level: HlsManifestRecoveryBurstLevel, +} + +impl HlsManifestRecoveryBurstConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsSegmentRepairSizeIncreaseConfigDto { + #[serde(default = "default_hls_segment_repair_low_size_increase_percent")] + pub low_percent: u8, + #[serde(default = "default_hls_segment_repair_medium_size_increase_percent")] + pub medium_percent: u8, + #[serde(default = "default_hls_segment_repair_high_size_increase_percent")] + pub high_percent: u8, +} + +impl Default for HlsSegmentRepairSizeIncreaseConfigDto { + fn default() -> Self { + Self { + low_percent: default_hls_segment_repair_low_size_increase_percent(), + medium_percent: default_hls_segment_repair_medium_size_increase_percent(), + high_percent: default_hls_segment_repair_high_size_increase_percent(), + } + } +} + +impl HlsSegmentRepairSizeIncreaseConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} + + fn validate(&self) -> Result<(), TuliproxError> { + let fields = [ + ("low_percent", self.low_percent), + ("medium_percent", self.medium_percent), + ("high_percent", self.high_percent), + ]; + for (field, value) in fields { + if value > 100 { + return Err(TuliproxError::ConfigReverseProxy(format!( + "hls_cache.segment_repair.size_increase.{field} must be <= 100" + ))); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsSegmentRepairConfigDto { + #[serde(default)] + pub max_level: HlsSegmentRepairMode, + #[serde(default = "default_hls_segment_repair_apply_to_first_segments")] + pub apply_to_first_segments: u8, + #[serde(default = "default_hls_segment_repair_max_parallel_repairs")] + pub max_parallel_repairs: usize, + #[serde(default = "default_hls_segment_repair_postprocess_timeout_ms")] + pub postprocess_timeout_ms: u64, + #[serde(default, skip_serializing_if = "HlsSegmentRepairSizeIncreaseConfigDto::is_empty")] + pub size_increase: HlsSegmentRepairSizeIncreaseConfigDto, + #[serde(default, skip_serializing_if = "HlsCorruptSegmentWatchdogConfigDto::is_empty")] + pub corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto, +} + +impl Default for HlsSegmentRepairConfigDto { + fn default() -> Self { + Self { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: default_hls_segment_repair_apply_to_first_segments(), + max_parallel_repairs: default_hls_segment_repair_max_parallel_repairs(), + postprocess_timeout_ms: default_hls_segment_repair_postprocess_timeout_ms(), + size_increase: HlsSegmentRepairSizeIncreaseConfigDto::default(), + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto::default(), + } + } +} + +impl HlsSegmentRepairConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub fn clean(&mut self) { + self.size_increase.clean(); + self.corrupt_segment_watchdog.clean(); + } + + fn validate(&self) -> Result<(), TuliproxError> { + if self.postprocess_timeout_ms < 100 { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.postprocess_timeout_ms must be >= 100".to_string(), + )); + } + self.size_increase.validate()?; + self.corrupt_segment_watchdog.validate() + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsCorruptSegmentWatchdogConfigDto { + #[serde(default)] + pub mode: HlsCorruptSegmentWatchdogMode, + #[serde(default = "default_hls_corrupt_segment_watchdog_max_parallel_jobs")] + pub max_parallel_jobs: usize, +} + +impl Default for HlsCorruptSegmentWatchdogConfigDto { + fn default() -> Self { + Self { + mode: HlsCorruptSegmentWatchdogMode::Off, + max_parallel_jobs: default_hls_corrupt_segment_watchdog_max_parallel_jobs(), + } + } +} + +impl HlsCorruptSegmentWatchdogConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} + + fn validate(&self) -> Result<(), TuliproxError> { + if self.max_parallel_jobs == 0 { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.corrupt_segment_watchdog.max_parallel_jobs must be >= 1".to_string(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsCacheConfigDto { + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub cache_path: Option, + #[serde(default)] + pub strip: HlsStripConfigDto, + #[serde(default = "default_hls_cache_duration")] + pub cache_duration: u64, + #[serde(default = "default_hls_cache_bytes")] + pub cache_bytes: ByteSize, + #[serde(default = "default_hls_cache_bytes_per_session")] + pub cache_bytes_per_session: ByteSize, + #[serde(default = "default_hls_max_segments_prefetch")] + pub max_segments_prefetch: usize, + #[serde(default = "default_hls_max_concurrent_segment_fetches_per_session")] + pub max_concurrent_segment_fetches_per_session: usize, + #[serde(default = "default_hls_max_concurrent_segment_fetches_global")] + pub max_concurrent_segment_fetches_global: usize, + #[serde(default = "default_hls_origin_manifest_timeout_ms")] + pub origin_manifest_timeout_ms: u64, + #[serde(default = "default_hls_origin_segment_timeout_ms")] + pub origin_segment_timeout_ms: u64, + #[serde(default = "default_hls_session_idle_timeout")] + pub session_idle_timeout: u64, + #[serde(default, skip_serializing_if = "HlsManifestRecoveryBurstConfigDto::is_empty")] + pub manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto, + #[serde(default, skip_serializing_if = "HlsSegmentRepairConfigDto::is_empty")] + pub segment_repair: HlsSegmentRepairConfigDto, +} + +impl Default for HlsCacheConfigDto { + fn default() -> Self { + Self { + cache_path: None, + strip: HlsStripConfigDto::default(), + cache_duration: default_hls_cache_duration(), + cache_bytes: default_hls_cache_bytes(), + cache_bytes_per_session: default_hls_cache_bytes_per_session(), + max_segments_prefetch: default_hls_max_segments_prefetch(), + max_concurrent_segment_fetches_per_session: default_hls_max_concurrent_segment_fetches_per_session(), + max_concurrent_segment_fetches_global: default_hls_max_concurrent_segment_fetches_global(), + origin_manifest_timeout_ms: default_hls_origin_manifest_timeout_ms(), + origin_segment_timeout_ms: default_hls_origin_segment_timeout_ms(), + session_idle_timeout: default_hls_session_idle_timeout(), + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto::default(), + segment_repair: HlsSegmentRepairConfigDto::default(), + } + } +} + +impl HlsCacheConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub fn clean(&mut self) { + self.strip.clean(); + self.manifest_recovery_burst.clean(); + self.segment_repair.clean(); + } + + fn ensure_min_u64(field_name: &str, value: u64, min_value: u64) -> Result<(), TuliproxError> { + if value < min_value { + return Err(TuliproxError::ConfigReverseProxy(format!("hls_cache.{field_name} must be >= {min_value}"))); + } + Ok(()) + } + + fn ensure_min_usize(field_name: &str, value: usize, min_value: usize) -> Result<(), TuliproxError> { + if value < min_value { + return Err(TuliproxError::ConfigReverseProxy(format!("hls_cache.{field_name} must be >= {min_value}"))); + } + Ok(()) + } + + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if let Some(cache_path) = &self.cache_path { + if cache_path.is_empty() { + self.cache_path = None; + } + } + + self.cache_bytes.clean_or_default(DEFAULT_HLS_CACHE_BYTES); + self.cache_bytes_per_session.clean_or_default(DEFAULT_HLS_CACHE_BYTES_PER_SESSION); + + self.cache_bytes.parse_bytes().map_err(TuliproxError::ConfigReverseProxy)?; + self.cache_bytes_per_session.parse_bytes().map_err(TuliproxError::ConfigReverseProxy)?; + + Self::ensure_min_u64("cache_duration", self.cache_duration, 1)?; + Self::ensure_min_usize( + "max_concurrent_segment_fetches_per_session", + self.max_concurrent_segment_fetches_per_session, + 1, + )?; + Self::ensure_min_usize("max_concurrent_segment_fetches_global", self.max_concurrent_segment_fetches_global, 1)?; + Self::ensure_min_u64("origin_manifest_timeout_ms", self.origin_manifest_timeout_ms, 1)?; + Self::ensure_min_u64("origin_segment_timeout_ms", self.origin_segment_timeout_ms, 1)?; + Self::ensure_min_u64("session_idle_timeout", self.session_idle_timeout, 1)?; + if self.segment_repair.apply_to_first_segments > 6 { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.apply_to_first_segments must be <= 6".to_string(), + )); + } + self.segment_repair.validate()?; + if self.segment_repair.max_level != HlsSegmentRepairMode::Off && self.segment_repair.max_parallel_repairs == 0 { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.max_parallel_repairs must be >= 1 when segment repair is enabled".to_string(), + )); + } + if self.segment_repair.max_level != HlsSegmentRepairMode::Off + && self.segment_repair.max_parallel_repairs > self.max_segments_prefetch + { + return Err(TuliproxError::ConfigReverseProxy(format!( + "hls_cache.segment_repair.max_parallel_repairs must be <= max_segments_prefetch ({})", + self.max_segments_prefetch + ))); + } + if self.segment_repair.corrupt_segment_watchdog.mode != HlsCorruptSegmentWatchdogMode::Off + && self.segment_repair.corrupt_segment_watchdog.max_parallel_jobs > self.max_segments_prefetch + { + return Err(TuliproxError::ConfigReverseProxy(format!( + "hls_cache.segment_repair.corrupt_segment_watchdog.max_parallel_jobs must be <= max_segments_prefetch ({})", + self.max_segments_prefetch + ))); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{ + ByteSize, HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsCorruptSegmentWatchdogMode, + HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairConfigDto, + HlsSegmentRepairMode, HlsSegmentRepairSizeIncreaseConfigDto, HlsStripConfigDto, HlsStripMode, + }; + use crate::model::ReverseProxyConfigDto; + + #[test] + fn hls_cache_deserializes_target_yaml() { + let yaml = r#" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + cache_path: + strip: + mode: "segments" + value: 0 + cache_duration: 300 + cache_bytes: "10GB" + cache_bytes_per_session: "512MB" + max_segments_prefetch: 6 + max_concurrent_segment_fetches_per_session: 2 + max_concurrent_segment_fetches_global: 64 + origin_manifest_timeout_ms: 3000 + origin_segment_timeout_ms: 10000 + session_idle_timeout: 300 +"#; + + let cfg: ReverseProxyConfigDto = serde_saphyr::from_str(yaml).expect("reverse_proxy should deserialize"); + assert_eq!(cfg.hls_cache, Some(HlsCacheConfigDto::default())); + } + + #[test] + fn hls_cache_serializes_under_reverse_proxy() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("hls_cache:"), "expected hls_cache block, got: {serialized}"); + } + + #[test] + fn hls_cache_serializes_non_default_segment_repair() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + apply_to_first_segments: 2, + max_parallel_repairs: 2, + postprocess_timeout_ms: 1_500, + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("segment_repair:"), "expected segment_repair block, got: {serialized}"); + assert!(serialized.contains("max_level: medium"), "expected segment repair max level, got: {serialized}"); + assert!( + !serialized.contains("mode: medium"), + "segment repair must serialize the v2 max_level field, got: {serialized}" + ); + assert!( + serialized.contains("apply_to_first_segments: 2"), + "expected segment repair first segment limit, got: {serialized}" + ); + assert!( + serialized.contains("max_parallel_repairs: 2"), + "expected segment repair parallel limit, got: {serialized}" + ); + assert!( + serialized.contains("postprocess_timeout_ms: 1500"), + "expected common post-processing timeout, got: {serialized}" + ); + } + + #[test] + fn hls_cache_serializes_non_default_corrupt_segment_watchdog_without_command_version() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto { + mode: HlsCorruptSegmentWatchdogMode::Sanitize, + max_parallel_jobs: 2, + }, + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("corrupt_segment_watchdog:"), "expected watchdog config block, got: {serialized}"); + assert!(serialized.contains("mode: sanitize"), "expected watchdog mode, got: {serialized}"); + assert!(serialized.contains("max_parallel_jobs: 2"), "expected watchdog parallel limit, got: {serialized}"); + assert!( + !serialized.contains("command_version"), + "command_version is internal and must not serialize, got: {serialized}" + ); + } + + #[test] + fn hls_cache_serializes_non_default_manifest_recovery_burst() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { + level: HlsManifestRecoveryBurstLevel::Beast, + }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!( + serialized.contains("manifest_recovery_burst:"), + "expected manifest recovery burst block, got: {serialized}" + ); + assert!(serialized.contains("level: beast"), "expected burst level, got: {serialized}"); + } + + #[test] + fn hls_cache_rejects_corrupt_segment_watchdog_command_version() { + let yaml = r#" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + segment_repair: + corrupt_segment_watchdog: + mode: sanitize + command_version: 1 +"#; + + let err = serde_saphyr::from_str::(yaml) + .expect_err("command_version must be rejected as unknown config"); + assert!(err.to_string().contains("command_version"), "unexpected error: {err}"); + } + + #[test] + fn hls_cache_serializes_non_default_segment_repair_size_increase() { + let segment_repair = HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + size_increase: HlsSegmentRepairSizeIncreaseConfigDto { medium_percent: 9, ..Default::default() }, + ..Default::default() + }; + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { segment_repair, ..Default::default() }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("size_increase:"), "expected size increase block, got: {serialized}"); + assert!(serialized.contains("medium_percent: 9"), "expected size increase value, got: {serialized}"); + } + + #[test] + fn hls_cache_with_non_default_segment_repair_is_not_empty() { + let cfg = HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + }, + ..Default::default() + }; + + assert!(!cfg.is_empty()); + } + + #[test] + fn hls_cache_rejects_unknown_fields() { + let yaml = r#" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + cache_path: "/tmp/tuliprox/cache/hls" + unknown_field: true +"#; + + let err = serde_saphyr::from_str::(yaml).expect_err("unknown fields must be rejected"); + assert!(err.to_string().contains("unknown_field"), "unexpected error: {err}"); + } + + #[test] + fn hls_cache_prepare_sets_defaults_for_blank_values() { + let mut cfg = HlsCacheConfigDto { + cache_path: None, + cache_bytes: ByteSize::new(" "), + cache_bytes_per_session: ByteSize::new(""), + ..Default::default() + }; + + cfg.prepare().expect("prepare should succeed"); + + assert_eq!(cfg, HlsCacheConfigDto::default()); + } + + #[test] + fn hls_cache_prepare_accepts_default_values() { + let mut cfg = HlsCacheConfigDto::default(); + + cfg.prepare().expect("default hls cache config should be valid"); + } + + #[test] + fn hls_cache_prepare_allows_zero_prefetch() { + let mut cfg = HlsCacheConfigDto { max_segments_prefetch: 0, ..Default::default() }; + + cfg.prepare().expect("zero prefetch should be valid"); + } + + #[test] + fn hls_cache_prepare_rejects_invalid_minimums() { + let cases: [(&str, HlsCacheConfigDto); 6] = [ + ("cache_duration", HlsCacheConfigDto { cache_duration: 0, ..Default::default() }), + ( + "max_concurrent_segment_fetches_per_session", + HlsCacheConfigDto { max_concurrent_segment_fetches_per_session: 0, ..Default::default() }, + ), + ( + "max_concurrent_segment_fetches_global", + HlsCacheConfigDto { max_concurrent_segment_fetches_global: 0, ..Default::default() }, + ), + ("origin_manifest_timeout_ms", HlsCacheConfigDto { origin_manifest_timeout_ms: 0, ..Default::default() }), + ("origin_segment_timeout_ms", HlsCacheConfigDto { origin_segment_timeout_ms: 0, ..Default::default() }), + ("session_idle_timeout", HlsCacheConfigDto { session_idle_timeout: 0, ..Default::default() }), + ]; + + for (field_name, mut cfg) in cases { + let err = cfg.prepare().expect_err("invalid minimum should be rejected"); + assert!(err.to_string().contains(field_name), "expected error to mention {field_name}, got: {err}"); + } + } + + #[test] + fn hls_cache_prepare_rejects_invalid_segment_repair_limits() { + let mut too_many_first_segments = HlsCacheConfigDto::default(); + too_many_first_segments.segment_repair.apply_to_first_segments = 7; + let err = too_many_first_segments.prepare().expect_err("too many first segments should be rejected"); + assert!(err.to_string().contains("segment_repair.apply_to_first_segments"), "unexpected error: {err}"); + + let mut too_many_parallel_repairs = HlsCacheConfigDto { max_segments_prefetch: 2, ..Default::default() }; + too_many_parallel_repairs.segment_repair.max_level = HlsSegmentRepairMode::Medium; + too_many_parallel_repairs.segment_repair.max_parallel_repairs = 3; + let err = too_many_parallel_repairs.prepare().expect_err("too many parallel repairs should be rejected"); + assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); + + let mut invalid_size_increase = HlsCacheConfigDto::default(); + invalid_size_increase.segment_repair.size_increase.high_percent = 101; + let err = invalid_size_increase.prepare().expect_err("size increase above 100 should be rejected"); + assert!(err.to_string().contains("segment_repair.size_increase.high_percent"), "unexpected error: {err}"); + + let mut invalid_postprocess_timeout = HlsCacheConfigDto::default(); + invalid_postprocess_timeout.segment_repair.postprocess_timeout_ms = 99; + let err = + invalid_postprocess_timeout.prepare().expect_err("post-processing timeout below 100ms should be rejected"); + assert!(err.to_string().contains("segment_repair.postprocess_timeout_ms"), "unexpected error: {err}"); + + let mut invalid_watchdog_parallel = HlsCacheConfigDto::default(); + invalid_watchdog_parallel.segment_repair.corrupt_segment_watchdog.max_parallel_jobs = 0; + let err = invalid_watchdog_parallel.prepare().expect_err("watchdog parallelism below 1 should be rejected"); + assert!( + err.to_string().contains("segment_repair.corrupt_segment_watchdog.max_parallel_jobs"), + "unexpected error: {err}" + ); + } + + #[test] + fn hls_cache_prepare_allows_zero_segment_repair_parallelism_when_repair_is_off() { + let mut cfg = HlsCacheConfigDto::default(); + cfg.segment_repair.max_parallel_repairs = 0; + + cfg.prepare().expect("zero repair parallelism is ignored while repair is off"); + } + + #[test] + fn hls_cache_prepare_rejects_zero_segment_repair_parallelism_when_repair_is_enabled() { + let mut cfg = HlsCacheConfigDto::default(); + cfg.segment_repair.max_level = HlsSegmentRepairMode::Medium; + cfg.segment_repair.max_parallel_repairs = 0; + + let err = cfg.prepare().expect_err("enabled repair needs a positive parallel limit"); + assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); + } + + #[test] + fn hls_strip_config_default_is_segments() { + let cfg = HlsStripConfigDto::default(); + assert_eq!(cfg.mode, HlsStripMode::Segments); + assert_eq!(cfg.value, 0); + } +} diff --git a/shared/src/model/config/hls_corrupt_segment_watchdog_mode.rs b/shared/src/model/config/hls_corrupt_segment_watchdog_mode.rs new file mode 100644 index 000000000..a19461e65 --- /dev/null +++ b/shared/src/model/config/hls_corrupt_segment_watchdog_mode.rs @@ -0,0 +1,33 @@ +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HlsCorruptSegmentWatchdogMode { + #[default] + Off, + DetectOnly, + Sanitize, + Diagnostic, +} + +impl HlsCorruptSegmentWatchdogMode { + /// True when the watchdog is configured to take any action (sanitize or + /// emit diagnostics). `Off` and `DetectOnly` are observation-only. + pub const fn is_enabled(self) -> bool { !matches!(self, Self::Off) } + + /// Stable lowercase log label — same canonical form as the `Display` impl + /// so log readers can correlate the two. + pub const fn as_log_value(self) -> &'static str { + match self { + Self::Off => "off", + Self::DetectOnly => "detect_only", + Self::Sanitize => "sanitize", + Self::Diagnostic => "diagnostic", + } + } +} + +crate::impl_str_enum!(HlsCorruptSegmentWatchdogMode, "HLS corrupt segment watchdog mode", + Off => "off", + DetectOnly => "detect_only", + Sanitize => "sanitize", + Diagnostic => "diagnostic", +); diff --git a/shared/src/model/config/hls_manifest_recovery_burst_level.rs b/shared/src/model/config/hls_manifest_recovery_burst_level.rs new file mode 100644 index 000000000..4b7f64e20 --- /dev/null +++ b/shared/src/model/config/hls_manifest_recovery_burst_level.rs @@ -0,0 +1,73 @@ +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HlsManifestRecoveryBurstLevel { + #[default] + Off, + Friendly, + Cautious, + Balanced, + Intense, + Aggressive, + Beast, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct HlsManifestRecoveryBurstPlan { + pub slots: usize, + pub lanes_per_slot: usize, +} + +impl HlsManifestRecoveryBurstPlan { + pub const fn total_candidates(self) -> usize { self.slots.saturating_mul(self.lanes_per_slot) } + + pub const fn slot_for_candidate(self, candidate_index: usize) -> usize { + match candidate_index.checked_div(self.lanes_per_slot) { + Some(slot) => slot, + None => 0, + } + } +} + +impl HlsManifestRecoveryBurstLevel { + /// Maps the configured burst level to a concrete fetch plan. The same + /// constants drive the runtime scheduler; see `HlsCacheConfig` for the + /// operator-facing description of each level. + pub const fn plan(self) -> HlsManifestRecoveryBurstPlan { + match self { + Self::Off => HlsManifestRecoveryBurstPlan { slots: 1, lanes_per_slot: 1 }, + Self::Friendly => HlsManifestRecoveryBurstPlan { slots: 2, lanes_per_slot: 1 }, + Self::Cautious => HlsManifestRecoveryBurstPlan { slots: 3, lanes_per_slot: 1 }, + Self::Balanced => HlsManifestRecoveryBurstPlan { slots: 4, lanes_per_slot: 1 }, + Self::Intense => HlsManifestRecoveryBurstPlan { slots: 5, lanes_per_slot: 1 }, + Self::Aggressive => HlsManifestRecoveryBurstPlan { slots: 6, lanes_per_slot: 1 }, + Self::Beast => HlsManifestRecoveryBurstPlan { slots: 6, lanes_per_slot: 2 }, + } + } + + pub const fn extra_candidates(self) -> usize { self.plan().total_candidates().saturating_sub(1) } + + pub const fn total_candidates(self) -> usize { self.plan().total_candidates() } +} + +crate::impl_str_enum!(HlsManifestRecoveryBurstLevel, "HLS manifest recovery burst level", + Off => "off", + Friendly => "friendly", + Cautious => "cautious", + Balanced => "balanced", + Intense => "intense", + Aggressive => "aggressive", + Beast => "beast", +); + +#[cfg(test)] +mod tests { + use super::HlsManifestRecoveryBurstLevel; + + #[test] + fn from_str_parses_known_levels() { + assert_eq!("off".parse::(), Ok(HlsManifestRecoveryBurstLevel::Off)); + assert_eq!("beast".parse::(), Ok(HlsManifestRecoveryBurstLevel::Beast)); + assert_eq!("balanced".parse::(), Ok(HlsManifestRecoveryBurstLevel::Balanced)); + assert!("nonsense".parse::().is_err()); + } +} diff --git a/shared/src/model/config/hls_segment_repair_mode.rs b/shared/src/model/config/hls_segment_repair_mode.rs new file mode 100644 index 000000000..60ba58a9e --- /dev/null +++ b/shared/src/model/config/hls_segment_repair_mode.rs @@ -0,0 +1,59 @@ +#[derive(Debug, Clone, Copy, Hash, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HlsSegmentRepairMode { + #[default] + Off, + Low, + Medium, + High, +} + +impl HlsSegmentRepairMode { + /// Stable lowercase log label — same canonical form as the `Display` impl. + pub const fn as_log_value(self) -> &'static str { + match self { + Self::Off => "off", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + } + } + + /// Ordering rank used by `execution_plan` to decide whether the configured + /// `max_level` covers the `required_level`. Higher number = more aggressive + /// repair. + pub const fn rank(self) -> u8 { + match self { + Self::Off => 0, + Self::Low => 1, + Self::Medium => 2, + Self::High => 3, + } + } + + /// Decide whether a repair at `self` (configured ceiling) should run for a + /// segment that requested `required_level` (e.g. by a trigger source). + pub const fn execution_plan(self, required_level: Self) -> HlsSegmentRepairExecutionPlan { + if matches!(self, Self::Off) || matches!(required_level, Self::Off) { + HlsSegmentRepairExecutionPlan::SkipNoTrigger + } else if self.rank() < required_level.rank() { + HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired + } else { + HlsSegmentRepairExecutionPlan::Repair(required_level) + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum HlsSegmentRepairExecutionPlan { + Repair(HlsSegmentRepairMode), + SkipNoTrigger, + SkipConfiguredMaxBelowRequired, +} + +crate::impl_str_enum!(HlsSegmentRepairMode, "HLS segment repair mode", + Off => "off", + Low => "low", + Medium => "medium", + High => "high", +); diff --git a/shared/src/model/config/hls_strip_mode.rs b/shared/src/model/config/hls_strip_mode.rs new file mode 100644 index 000000000..df1f97e3b --- /dev/null +++ b/shared/src/model/config/hls_strip_mode.rs @@ -0,0 +1,12 @@ +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HlsStripMode { + #[default] + Segments, + Seconds, +} + +crate::impl_str_enum!(HlsStripMode, "HLS strip mode", + Segments => "segments", + Seconds => "seconds", +); diff --git a/shared/src/model/config/input.rs b/shared/src/model/config/input.rs index 4326800f9..a2a8e649e 100644 --- a/shared/src/model/config/input.rs +++ b/shared/src/model/config/input.rs @@ -1,15 +1,17 @@ use super::PanelApiConfigDto; use crate::{ check_input_connections, check_input_credentials, + defaults::{ + default_as_true, default_probe_delay_secs, default_probe_live_interval, default_resolve_background, + default_resolve_delay_secs, default_xtream_live_stream_use_prefix, is_default_probe_delay_secs, + is_default_probe_live_interval, is_default_resolve_delay_secs, is_false, is_true, is_zero_i16, is_zero_u16, + }, error::TuliproxError, foundation::{get_filter, Filter}, model::{config::media_server_catalog::MediaServerInputConfigDto, EpgConfigDto, PatternTemplate}, utils::{ - arc_str_serde, arc_str_vec_serde, default_as_true, default_probe_delay_secs, default_probe_live_interval, - default_resolve_background, default_resolve_delay_secs, default_xtream_live_stream_use_prefix, - deserialize_timestamp, get_credentials_from_url_str, get_trimmed_string, is_blank_optional_string, - is_default_probe_delay_secs, is_default_probe_live_interval, is_default_resolve_delay_secs, is_false, - is_non_blank_optional_string, is_true, is_zero_i16, is_zero_u16, parse_duration_seconds, + arc_str_serde, arc_str_vec_serde, deserialize_timestamp, get_credentials_from_url_str, get_trimmed_string, + is_blank_optional_string, is_non_blank_optional_string, parse_duration_seconds, parse_provider_scheme_url_parts, sanitize_sensitive_info, serialize_option_vec_flow_map_items, trim_last_slash, Internable, BATCH_SCHEME_PREFIX, PROVIDER_SCHEME_PREFIX, }, diff --git a/shared/src/model/config/library.rs b/shared/src/model/config/library.rs index 074ecbbe5..7f2e012cd 100644 --- a/shared/src/model/config/library.rs +++ b/shared/src/model/config/library.rs @@ -1,10 +1,10 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_as_true, default_movie_category, default_series_category, default_storage_formats, default_supported_library_extensions, default_thumbnail_height, default_thumbnail_width, is_default_supported_library_extensions, is_true, }, + error::TuliproxError, }; use serde::{Deserialize, Serialize}; diff --git a/shared/src/model/config/log.rs b/shared/src/model/config/log.rs index 38764025d..fc4005fb8 100644 --- a/shared/src/model/config/log.rs +++ b/shared/src/model/config/log.rs @@ -1,6 +1,6 @@ -use crate::utils::{ - default_as_true, is_blank_optional_str, is_blank_optional_string, is_default_runtime_config_report_format, - is_false, is_true, +use crate::{ + defaults::{default_as_true, is_default_runtime_config_report_format, is_false, is_true}, + utils::{is_blank_optional_str, is_blank_optional_string}, }; use strum_macros::{AsRefStr, Display, EnumIter, EnumString}; diff --git a/shared/src/model/config/media_server_catalog.rs b/shared/src/model/config/media_server_catalog.rs index 3ca5fd26b..d2a8af549 100644 --- a/shared/src/model/config/media_server_catalog.rs +++ b/shared/src/model/config/media_server_catalog.rs @@ -1,11 +1,10 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_as_true, default_media_server_catalog_page_size, default_media_server_catalog_request_delay_ms, - deserialize_as_option_string, get_trimmed_string, is_blank_optional_string, - is_default_media_server_catalog_page_size, is_default_media_server_catalog_request_delay_ms, is_false, - is_non_blank_optional_string, is_true, + is_default_media_server_catalog_page_size, is_default_media_server_catalog_request_delay_ms, is_false, is_true, }, + error::TuliproxError, + utils::{deserialize_as_option_string, get_trimmed_string, is_blank_optional_string, is_non_blank_optional_string}, }; use std::sync::Arc; diff --git a/shared/src/model/config/messaging.rs b/shared/src/model/config/messaging.rs index 47e8a4e90..b9c0fbca9 100644 --- a/shared/src/model/config/messaging.rs +++ b/shared/src/model/config/messaging.rs @@ -1,10 +1,8 @@ use crate::{ + defaults::{default_critical_percent, default_repeat_interval_secs, default_warn_percent, is_false}, error::TuliproxError, model::MsgKind, - utils::{ - default_critical_percent, default_repeat_interval_secs, default_warn_percent, is_blank_optional_str, - is_blank_optional_string, is_false, - }, + utils::{is_blank_optional_str, is_blank_optional_string}, }; #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] diff --git a/shared/src/model/config/metadata_update.rs b/shared/src/model/config/metadata_update.rs index bf679612d..afe78042e 100644 --- a/shared/src/model/config/metadata_update.rs +++ b/shared/src/model/config/metadata_update.rs @@ -1,6 +1,5 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_metadata_backoff_jitter_percent, default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_max_attempts_probe, @@ -13,22 +12,24 @@ use crate::{ default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, default_probe_user_priority, default_tmdb_api_key, default_tmdb_cache_duration_days, default_tmdb_language, - default_tmdb_match_threshold, default_tmdb_rate_limit_ms, deserialize_as_string, - is_default_metadata_backoff_jitter_percent, is_default_metadata_ffprobe_analyze_duration, - is_default_metadata_ffprobe_live_analyze_duration, is_default_metadata_ffprobe_live_probe_size, - is_default_metadata_ffprobe_probe_size, is_default_metadata_max_attempts_probe, - is_default_metadata_max_attempts_resolve, is_default_metadata_max_queue_size, - is_default_metadata_max_resolve_retry_backoff, is_default_metadata_no_change_cache_ttl_secs, - is_default_metadata_path, is_default_metadata_probe_cooldown, is_default_metadata_probe_fairness_resolve_burst, - is_default_metadata_probe_retry_backoff_step_1, is_default_metadata_probe_retry_backoff_step_2, - is_default_metadata_probe_retry_backoff_step_3, is_default_metadata_probe_retry_load_retry_delay, - is_default_metadata_progress_log_interval, is_default_metadata_queue_log_interval, - is_default_metadata_resolve_exhaustion_reset_gap, is_default_metadata_resolve_min_retry_base, - is_default_metadata_retry_delay, is_default_metadata_tmdb_cooldown, is_default_metadata_worker_idle_timeout, - is_default_probe_user_priority, is_default_tmdb_cache_duration_days, is_default_tmdb_language, - is_default_tmdb_match_threshold, is_default_tmdb_rate_limit_ms, is_false, is_tmdb_default_api_key, - parse_duration_seconds, parse_size_base_2, TMDB_API_KEY, + default_tmdb_match_threshold, default_tmdb_rate_limit_ms, is_default_metadata_backoff_jitter_percent, + is_default_metadata_ffprobe_analyze_duration, is_default_metadata_ffprobe_live_analyze_duration, + is_default_metadata_ffprobe_live_probe_size, is_default_metadata_ffprobe_probe_size, + is_default_metadata_max_attempts_probe, is_default_metadata_max_attempts_resolve, + is_default_metadata_max_queue_size, is_default_metadata_max_resolve_retry_backoff, + is_default_metadata_no_change_cache_ttl_secs, is_default_metadata_path, is_default_metadata_probe_cooldown, + is_default_metadata_probe_fairness_resolve_burst, is_default_metadata_probe_retry_backoff_step_1, + is_default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_3, + is_default_metadata_probe_retry_load_retry_delay, is_default_metadata_progress_log_interval, + is_default_metadata_queue_log_interval, is_default_metadata_resolve_exhaustion_reset_gap, + is_default_metadata_resolve_min_retry_base, is_default_metadata_retry_delay, is_default_metadata_tmdb_cooldown, + is_default_metadata_worker_idle_timeout, is_default_probe_user_priority, is_default_tmdb_cache_duration_days, + is_default_tmdb_language, is_default_tmdb_match_threshold, is_default_tmdb_rate_limit_ms, is_false, + is_tmdb_default_api_key, TMDB_API_KEY, }, + error::TuliproxError, + model::ByteSize, + utils::{deserialize_as_string, parse_duration_seconds}, }; const MIN_DURATION_SECS: u64 = 1; @@ -338,10 +339,9 @@ pub struct FfprobeConfigDto { pub analyze_duration: String, #[serde( default = "default_metadata_ffprobe_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_probe_size", - deserialize_with = "deserialize_as_string" + skip_serializing_if = "is_default_metadata_ffprobe_probe_size" )] - pub probe_size: String, + pub probe_size: ByteSize, #[serde( default = "default_metadata_ffprobe_live_analyze_duration", skip_serializing_if = "is_default_metadata_ffprobe_live_analyze_duration", @@ -350,10 +350,9 @@ pub struct FfprobeConfigDto { pub live_analyze_duration: String, #[serde( default = "default_metadata_ffprobe_live_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size", - deserialize_with = "deserialize_as_string" + skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size" )] - pub live_probe_size: String, + pub live_probe_size: ByteSize, } impl Default for FfprobeConfigDto { @@ -386,13 +385,13 @@ impl FfprobeConfigDto { if self.analyze_duration.trim().is_empty() { self.analyze_duration = default_metadata_ffprobe_analyze_duration(); } - if self.probe_size.trim().is_empty() { + if self.probe_size.as_str().trim().is_empty() { self.probe_size = default_metadata_ffprobe_probe_size(); } if self.live_analyze_duration.trim().is_empty() { self.live_analyze_duration = default_metadata_ffprobe_live_analyze_duration(); } - if self.live_probe_size.trim().is_empty() { + if self.live_probe_size.as_str().trim().is_empty() { self.live_probe_size = default_metadata_ffprobe_live_probe_size(); } } @@ -407,12 +406,14 @@ impl FfprobeConfigDto { )?; self.analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(analyze_duration_secs); - let probe_size_bytes = parse_size_base_2(&self.probe_size) + let probe_size_bytes = self + .probe_size + .parse_bytes() .map_err(|err| { TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.probe_size`: {err}")) })? .max(1); - self.probe_size = MetadataUpdateConfigDto::canonicalize_size_bytes(probe_size_bytes); + self.probe_size = ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(probe_size_bytes)); let live_analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( &self.live_analyze_duration, @@ -421,12 +422,14 @@ impl FfprobeConfigDto { )?; self.live_analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(live_analyze_duration_secs); - let live_probe_size_bytes = parse_size_base_2(&self.live_probe_size) + let live_probe_size_bytes = self + .live_probe_size + .parse_bytes() .map_err(|err| { TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.live_probe_size`: {err}")) })? .max(1); - self.live_probe_size = MetadataUpdateConfigDto::canonicalize_size_bytes(live_probe_size_bytes); + self.live_probe_size = ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(live_probe_size_bytes)); Ok(()) } @@ -605,7 +608,7 @@ impl MetadataUpdateConfigDto { #[cfg(test)] mod tests { - use super::MetadataUpdateConfigDto; + use super::{ByteSize, MetadataUpdateConfigDto}; #[test] fn default_config_is_empty() { @@ -647,7 +650,7 @@ mod tests { cfg.probe_fairness_resolve_burst = 0; cfg.ffprobe.timeout = Some(0); cfg.ffprobe.analyze_duration = "0s".to_string(); - cfg.ffprobe.probe_size = "0".to_string(); + cfg.ffprobe.probe_size = ByteSize::new("0"); cfg.prepare().expect("metadata update config should clamp minimum values"); @@ -659,7 +662,7 @@ mod tests { assert_eq!(cfg.probe_fairness_resolve_burst, 1); assert_eq!(cfg.ffprobe.timeout, Some(1)); assert_eq!(cfg.ffprobe.analyze_duration, "1s"); - assert_eq!(cfg.ffprobe.probe_size, "1B"); + assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("1B")); } #[test] @@ -679,9 +682,9 @@ mod tests { cfg.worker_idle_timeout = "60".to_string(); cfg.probe.retry_backoff_step_3 = "3600".to_string(); cfg.ffprobe.analyze_duration = "10s".to_string(); - cfg.ffprobe.probe_size = "10485760".to_string(); + cfg.ffprobe.probe_size = ByteSize::new("10485760"); cfg.ffprobe.live_analyze_duration = "5s".to_string(); - cfg.ffprobe.live_probe_size = "5242880".to_string(); + cfg.ffprobe.live_probe_size = ByteSize::new("5242880"); cfg.prepare().expect("metadata update config should canonicalize durations"); @@ -690,9 +693,9 @@ mod tests { assert_eq!(cfg.worker_idle_timeout, "1m"); assert_eq!(cfg.probe.retry_backoff_step_3, "1h"); assert_eq!(cfg.ffprobe.analyze_duration, "10s"); - assert_eq!(cfg.ffprobe.probe_size, "10MB"); + assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("10MB")); assert_eq!(cfg.ffprobe.live_analyze_duration, "5s"); - assert_eq!(cfg.ffprobe.live_probe_size, "5MB"); + assert_eq!(cfg.ffprobe.live_probe_size, ByteSize::new("5MB")); } #[test] diff --git a/shared/src/model/config/mod.rs b/shared/src/model/config/mod.rs index 72c97e90f..d9ece2cb7 100644 --- a/shared/src/model/config/mod.rs +++ b/shared/src/model/config/mod.rs @@ -3,12 +3,18 @@ mod api_proxy; mod api_user; mod app_config; mod base; +mod byte_size; mod cache; mod config_type; mod epg; mod epg_smart_match; mod geoip; mod hdhomerun; +mod hls; +mod hls_corrupt_segment_watchdog_mode; +mod hls_manifest_recovery_burst_level; +mod hls_segment_repair_mode; +mod hls_strip_mode; mod input; mod ipcheck; mod log; @@ -50,6 +56,7 @@ pub use api_proxy::*; pub use api_user::*; pub use app_config::*; pub use base::*; +pub use byte_size::*; pub use cache::*; pub use config_type::*; pub use disk_alert::*; @@ -58,6 +65,11 @@ pub use epg_smart_match::*; pub use favourites::*; pub use geoip::*; pub use hdhomerun::*; +pub use hls::*; +pub use hls_corrupt_segment_watchdog_mode::*; +pub use hls_manifest_recovery_burst_level::*; +pub use hls_segment_repair_mode::*; +pub use hls_strip_mode::*; pub use input::*; pub use ipcheck::*; pub use library::*; diff --git a/shared/src/model/config/panel_api.rs b/shared/src/model/config/panel_api.rs index a98e3a60f..750c9fb0f 100644 --- a/shared/src/model/config/panel_api.rs +++ b/shared/src/model/config/panel_api.rs @@ -1,10 +1,13 @@ use crate::{ + defaults::{ + default_as_true, default_panel_api_alias_pool_max, default_panel_api_alias_pool_min, + default_panel_api_provision_cooldown_secs, default_panel_api_provision_probe_interval_secs, + default_panel_api_provision_timeout_secs, is_true, + }, error::TuliproxError, utils::{ - arc_str_option_serde, arc_str_serde, default_as_true, default_panel_api_alias_pool_max, - default_panel_api_alias_pool_min, default_panel_api_provision_cooldown_secs, - default_panel_api_provision_probe_interval_secs, default_panel_api_provision_timeout_secs, - deserialize_as_option_string, is_blank_optional_arc_str, is_true, serialize_vec_flow_map_items, + arc_str_option_serde, arc_str_serde, deserialize_as_option_string, is_blank_optional_arc_str, + serialize_vec_flow_map_items, }, }; use log::warn; diff --git a/shared/src/model/config/paths.rs b/shared/src/model/config/paths.rs index e4264a550..1a1f69d9c 100644 --- a/shared/src/model/config/paths.rs +++ b/shared/src/model/config/paths.rs @@ -1,4 +1,4 @@ -use crate::utils::{is_blank_optional_string, is_empty_optional_vec}; +use crate::{defaults::is_empty_optional_vec, utils::is_blank_optional_string}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ConfigPaths { diff --git a/shared/src/model/config/qos_aggregation.rs b/shared/src/model/config/qos_aggregation.rs index ba4f791ab..a9bf9d7c1 100644 --- a/shared/src/model/config/qos_aggregation.rs +++ b/shared/src/model/config/qos_aggregation.rs @@ -1,4 +1,4 @@ -use crate::{error::TuliproxError, utils::is_false}; +use crate::{defaults::is_false, error::TuliproxError}; const fn default_qos_aggregation_interval_secs() -> u64 { 300 } const fn is_default_qos_aggregation_interval_secs(value: &u64) -> bool { diff --git a/shared/src/model/config/reverse_proxy.rs b/shared/src/model/config/reverse_proxy.rs index 7a243fec7..ea158009e 100644 --- a/shared/src/model/config/reverse_proxy.rs +++ b/shared/src/model/config/reverse_proxy.rs @@ -1,14 +1,15 @@ use crate::{ - error::TuliproxError, - model::{ - CacheConfigDto, GeoIpConfigDto, QosAggregationConfigDto, RateLimitConfigDto, StreamConfigDto, - StreamHistoryConfigDto, - }, - utils::{ + defaults::{ default_resource_retry_attempts, default_resource_retry_backoff_ms, default_resource_retry_backoff_multiplier, - hex_to_u8_16, is_default_resource_retry_attempts, is_default_resource_retry_backoff_ms, + is_default_resource_retry_attempts, is_default_resource_retry_backoff_ms, is_default_resource_retry_backoff_multiplier, is_empty_optional_vec, is_false, }, + error::TuliproxError, + model::{ + CacheConfigDto, GeoIpConfigDto, HlsCacheConfigDto, QosAggregationConfigDto, RateLimitConfigDto, + StreamConfigDto, StreamHistoryConfigDto, + }, + utils::hex_to_u8_16, }; use log::warn; @@ -58,6 +59,8 @@ pub struct ReverseProxyConfigDto { pub stream_history: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub qos_aggregation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hls_cache: Option, } impl ReverseProxyConfigDto { @@ -72,6 +75,7 @@ impl ReverseProxyConfigDto { && (self.stream_history.is_none() || self.stream_history.as_ref().is_some_and(|s| s.is_empty())) && (self.qos_aggregation.is_none() || self.qos_aggregation.as_ref().is_some_and(QosAggregationConfigDto::is_empty)) + && (self.hls_cache.is_none() || self.hls_cache.as_ref().is_some_and(HlsCacheConfigDto::is_empty)) } pub fn clean(&mut self) { @@ -102,6 +106,12 @@ impl ReverseProxyConfigDto { if self.qos_aggregation.as_ref().is_some_and(QosAggregationConfigDto::is_empty) { self.qos_aggregation = None; } + if let Some(hls_cache) = self.hls_cache.as_mut() { + hls_cache.clean(); + if hls_cache.is_empty() { + self.hls_cache = None; + } + } } pub(crate) fn prepare(&mut self, storage_dir: &str) -> Result<(), TuliproxError> { @@ -144,6 +154,9 @@ impl ReverseProxyConfigDto { if let Some(resource_retry) = self.resource_retry.as_mut() { resource_retry.prepare()?; } + if let Some(hls_cache) = self.hls_cache.as_mut() { + hls_cache.prepare()?; + } Ok(()) } @@ -180,11 +193,15 @@ impl Default for ResourceRetryConfigDto { } impl ResourceRetryConfigDto { + /// True when every field matches its documented default and the user has not + /// written an explicit (even if empty) `failover_redirect_patterns` section. + /// `Some(vec![])` is preserved on purpose — writing `failover_redirect_patterns: []` + /// signals "I have a retry block, just no patterns" and dropping it would lose intent. pub fn is_default(&self) -> bool { self.max_attempts == default_resource_retry_attempts() && self.backoff_millis == default_resource_retry_backoff_ms() && (self.backoff_multiplier - default_resource_retry_backoff_multiplier()).abs() < f64::EPSILON - && is_empty_optional_vec(&self.failover_redirect_patterns) + && self.failover_redirect_patterns.is_none() } pub fn prepare(&mut self) -> Result<(), TuliproxError> { @@ -201,8 +218,10 @@ impl ResourceRetryConfigDto { #[cfg(test)] mod tests { - use super::ReverseProxyConfigDto; - use crate::model::{QosAggregationConfigDto, StreamHistoryConfigDto}; + use crate::model::{ + ByteSize, HlsCacheConfigDto, HlsSegmentRepairConfigDto, HlsSegmentRepairMode, QosAggregationConfigDto, + ReverseProxyConfigDto, StreamHistoryConfigDto, + }; #[test] fn serializing_stream_history_under_reverse_proxy_uses_nested_yaml_shape() { @@ -317,4 +336,56 @@ qos_aggregation: let err = cfg.prepare("storage").expect_err("prepare must reject zero interval"); assert!(err.to_string().contains("interval_secs"), "unexpected error: {err}"); } + + #[test] + fn reverse_proxy_prepare_rejects_invalid_hls_cache_byte_size() { + let mut cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { cache_bytes: ByteSize::new("10XB"), ..Default::default() }), + ..Default::default() + }; + + let err = cfg.prepare("storage").expect_err("invalid hls cache byte size must be rejected"); + + assert!(err.to_string().contains("Invalid size"), "unexpected error: {err}"); + } + + #[test] + fn reverse_proxy_clean_removes_default_hls_cache() { + let mut cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + }; + + cfg.clean(); + + assert!(cfg.hls_cache.is_none()); + } + + #[test] + fn reverse_proxy_clean_keeps_non_default_hls_segment_repair() { + let mut cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + cfg.clean(); + + let Some(hls_cache) = cfg.hls_cache else { + panic!("non-default hls segment repair must keep hls_cache"); + }; + assert_eq!(hls_cache.segment_repair.max_level, HlsSegmentRepairMode::Medium); + assert_eq!(hls_cache.segment_repair.apply_to_first_segments, 1); + assert_eq!(hls_cache.segment_repair.max_parallel_repairs, 1); + } } diff --git a/shared/src/model/config/source.rs b/shared/src/model/config/source.rs index 9a9e43c38..b79fee74a 100644 --- a/shared/src/model/config/source.rs +++ b/shared/src/model/config/source.rs @@ -1,10 +1,11 @@ use crate::{ + defaults::default_as_default, error::TuliproxError, foundation::prepare_templates, model::{ config::target::ConfigTargetDto, ConfigInputDto, ConfigProviderDto, HdHomeRunDeviceOverview, PatternTemplate, }, - utils::{arc_str_vec_serde, default_as_default, Internable}, + utils::{arc_str_vec_serde, Internable}, }; use log::warn; use std::{collections::HashSet, sync::Arc}; diff --git a/shared/src/model/config/stream.rs b/shared/src/model/config/stream.rs index d409b5219..1b26ab8ca 100644 --- a/shared/src/model/config/stream.rs +++ b/shared/src/model/config/stream.rs @@ -1,12 +1,12 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_as_true, default_catchup_session_ttl_secs, default_grace_period_millis, default_grace_period_timeout_secs, default_hls_session_ttl_secs, default_shared_burst_buffer_mb, - is_blank_optional_string, is_default_catchup_session_ttl_secs, is_default_grace_period_millis, - is_default_grace_period_timeout_secs, is_default_hls_session_ttl_secs, is_default_shared_burst_buffer_mb, - is_true, parse_to_kbps, + is_default_catchup_session_ttl_secs, is_default_grace_period_millis, is_default_grace_period_timeout_secs, + is_default_hls_session_ttl_secs, is_default_shared_burst_buffer_mb, is_false, is_true, }, + error::TuliproxError, + utils::{is_blank_optional_string, parse_to_kbps}, }; use std::{ fmt::{Display, Formatter}, @@ -94,7 +94,7 @@ impl StreamBufferConfigDto { pub struct StreamConfigDto { #[serde(default = "default_as_true", skip_serializing_if = "is_true")] pub retry: bool, - #[serde(default, skip_serializing_if = "crate::utils::is_false")] + #[serde(default, skip_serializing_if = "is_false")] pub metrics_enabled: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub buffer: Option, diff --git a/shared/src/model/config/stream_history.rs b/shared/src/model/config/stream_history.rs index 898cc0100..514eb0868 100644 --- a/shared/src/model/config/stream_history.rs +++ b/shared/src/model/config/stream_history.rs @@ -1,10 +1,10 @@ use crate::{ - error::TuliproxError, - utils::{ + defaults::{ default_stream_history_batch_size, default_stream_history_retention_days, is_blank_stream_history_directory, is_default_stream_history_batch_size, is_default_stream_history_retention_days, is_false, DEFAULT_STREAM_HISTORY_DIR, }, + error::TuliproxError, }; use std::path::PathBuf; diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 4f7be7dca..98ea3201e 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -1,4 +1,8 @@ use crate::{ + defaults::{ + default_as_default, default_as_true, is_config_target_options_empty, is_default_processing_order, is_false, + is_true, is_zero_u16, + }, error::TuliproxError, foundation::{get_filter, Filter}, handle_tuliprox_error_result_list, @@ -6,19 +10,52 @@ use crate::{ ClusterFlags, ConfigFavouritesDto, ConfigRenameDto, ConfigSortDto, HdHomeRunDeviceOverview, PatternTemplate, ProcessingOrder, StrmExportStyle, TargetType, TraktConfigDto, }, - utils::{ - default_as_default, default_as_true, is_blank_optional_string, is_config_target_options_empty, - is_default_processing_order, is_false, is_true, is_zero_u16, - }, + utils::is_blank_optional_string, }; +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigTargetShareLiveStreams { + #[serde(default, skip_serializing_if = "is_false")] + pub hls: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub mpeg_ts: bool, +} + +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum ConfigTargetShareLiveStreamsCompat { + Legacy(bool), + Structured(ConfigTargetShareLiveStreams), +} + +fn deserialize_share_live_streams<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(match ::deserialize(deserializer)? { + ConfigTargetShareLiveStreamsCompat::Legacy(enabled) => { + ConfigTargetShareLiveStreams { hls: enabled, mpeg_ts: enabled } + } + ConfigTargetShareLiveStreamsCompat::Structured(config) => config, + }) +} + +impl ConfigTargetShareLiveStreams { + pub fn is_empty(&self) -> bool { !self.hls && !self.mpeg_ts } +} + #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigTargetOptions { #[serde(default, skip_serializing_if = "is_false")] pub ignore_logo: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub share_live_streams: bool, + #[serde( + default, + deserialize_with = "deserialize_share_live_streams", + skip_serializing_if = "ConfigTargetShareLiveStreams::is_empty" + )] + pub share_live_streams: ConfigTargetShareLiveStreams, #[serde(default, skip_serializing_if = "is_false")] pub remove_duplicates: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -28,11 +65,17 @@ pub struct ConfigTargetOptions { impl ConfigTargetOptions { pub fn is_empty(&self) -> bool { !self.ignore_logo - && !self.share_live_streams + && self.share_live_streams.is_empty() && !self.remove_duplicates && (self.force_redirect.is_none() || self.force_redirect.is_some_and(|f| f.has_full_flags() || f.is_empty())) } + + pub fn share_live_hls_enabled(&self) -> bool { self.share_live_streams.hls } + + pub fn share_live_mpeg_ts_enabled(&self) -> bool { self.share_live_streams.mpeg_ts } + + pub fn share_live_any_enabled(&self) -> bool { self.share_live_hls_enabled() || self.share_live_mpeg_ts_enabled() } } #[allow(clippy::struct_excessive_bools)] @@ -431,7 +474,10 @@ impl ConfigTargetDto { #[cfg(test)] mod tests { - use super::{ConfigTargetDto, M3uTargetOutputDto, StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto}; + use super::{ + ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, M3uTargetOutputDto, StrmTargetOutputDto, + TargetOutputDto, XtreamTargetOutputDto, + }; fn target_with_outputs(output: Vec) -> ConfigTargetDto { ConfigTargetDto { @@ -499,4 +545,63 @@ mod tests { assert!(err.to_string().contains("xtream or m3u output")); } + + #[test] + fn target_options_deserialize_structured_share_live_streams() { + let yaml = r#" +share_live_streams: + hls: true + mpeg_ts: true +"#; + + let options: ConfigTargetOptions = + serde_saphyr::from_str(yaml).expect("structured share_live_streams should deserialize"); + + assert!(options.share_live_hls_enabled()); + assert!(options.share_live_mpeg_ts_enabled()); + assert!(options.share_live_any_enabled()); + } + + #[test] + fn target_options_maps_legacy_true_share_live_streams_to_both_modes() { + let yaml = r#" +share_live_streams: true +"#; + + let options = serde_saphyr::from_str::(yaml); + + assert!(options.is_ok(), "legacy boolean should deserialize: {options:?}"); + if let Ok(options) = options { + assert_eq!(options.share_live_streams, ConfigTargetShareLiveStreams { hls: true, mpeg_ts: true }); + } + } + + #[test] + fn target_options_omit_default_share_live_streams() { + let options = ConfigTargetOptions::default(); + + assert!(options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("default options should serialize"); + assert!( + !serialized.contains("share_live_streams"), + "default share_live_streams should be omitted, got: {serialized}" + ); + } + + #[test] + fn target_options_mpeg_ts_helper_keeps_existing_stream_share_semantics() { + let hls_only = ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, + ..Default::default() + }; + let mpeg_ts = ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: false, mpeg_ts: true }, + ..Default::default() + }; + + assert!(hls_only.share_live_hls_enabled()); + assert!(!hls_only.share_live_mpeg_ts_enabled()); + assert!(mpeg_ts.share_live_mpeg_ts_enabled()); + } } diff --git a/shared/src/model/config/trakt.rs b/shared/src/model/config/trakt.rs index f5e67c656..fb1da49a2 100644 --- a/shared/src/model/config/trakt.rs +++ b/shared/src/model/config/trakt.rs @@ -1,4 +1,4 @@ -use crate::utils::{ +use crate::defaults::{ default_as_true, default_trakt_fuzzy_threshold, is_false, is_true, DEFAULT_USER_AGENT, TRAKT_API_KEY, TRAKT_API_URL, TRAKT_API_VERSION, }; diff --git a/shared/src/model/config/video_download.rs b/shared/src/model/config/video_download.rs index cc17ebb37..a804a7d58 100644 --- a/shared/src/model/config/video_download.rs +++ b/shared/src/model/config/video_download.rs @@ -1,10 +1,11 @@ use crate::{ - error::TuliproxError, - utils::{ - default_download_dir, default_episode_pattern, default_supported_video_extensions, is_blank_optional_str, - is_blank_optional_string, is_blank_or_default_download_dir, is_blank_or_default_episode_pattern, - is_default_supported_video_extensions, is_false, DEFAULT_USER_AGENT, F64_DEFAULT_EPSILON, + defaults::{ + default_download_dir, default_episode_pattern, default_supported_video_extensions, + is_blank_or_default_download_dir, is_blank_or_default_episode_pattern, is_default_supported_video_extensions, + is_false, DEFAULT_USER_AGENT, F64_DEFAULT_EPSILON, }, + error::TuliproxError, + utils::{is_blank_optional_str, is_blank_optional_string}, }; use std::{borrow::BorrowMut, collections::HashMap}; @@ -152,7 +153,7 @@ impl VideoConfigDto { #[cfg(test)] mod tests { use super::*; - use crate::utils::DEFAULT_DOWNLOAD_DIR; + use crate::defaults::DEFAULT_DOWNLOAD_DIR; fn make_test_download_config() -> VideoDownloadConfigDto { VideoDownloadConfigDto { diff --git a/shared/src/model/config/web_auth.rs b/shared/src/model/config/web_auth.rs index 5a5693f94..cf5bf3f35 100644 --- a/shared/src/model/config/web_auth.rs +++ b/shared/src/model/config/web_auth.rs @@ -1,4 +1,4 @@ -use crate::utils::{ +use crate::defaults::{ default_as_true, default_token_ttl_mins, default_user_file_path, default_user_group_file_path, is_blank_or_default_user_file_path, is_blank_or_default_user_group_file_path, is_default_token_ttl_mins, is_true, }; diff --git a/shared/src/model/config/web_ui.rs b/shared/src/model/config/web_ui.rs index 8360e3c15..e1284566a 100644 --- a/shared/src/model/config/web_ui.rs +++ b/shared/src/model/config/web_ui.rs @@ -1,11 +1,9 @@ use crate::{ apply_flags, create_bitset, + defaults::{default_as_true, default_kick_secs, is_default_kick_secs, is_false, is_true}, error::TuliproxError, model::{view_type::ViewType, WebAuthConfigDto}, - utils::{ - default_as_true, default_kick_secs, is_blank_optional_str, is_blank_optional_string, is_default_kick_secs, - is_false, is_true, - }, + utils::{is_blank_optional_str, is_blank_optional_string}, }; const RESERVED_PATHS: &[&str] = &[ diff --git a/shared/src/model/pagination.rs b/shared/src/model/pagination.rs index 750563964..36a155904 100644 --- a/shared/src/model/pagination.rs +++ b/shared/src/model/pagination.rs @@ -1,4 +1,4 @@ -use crate::utils::{default_page, default_page_size}; +use crate::defaults::{default_page, default_page_size}; use serde::{Deserialize, Serialize}; /// Search mode for paged queries. diff --git a/shared/src/model/stream_history.rs b/shared/src/model/stream_history.rs index 1885d5a88..0398cc435 100644 --- a/shared/src/model/stream_history.rs +++ b/shared/src/model/stream_history.rs @@ -1,6 +1,6 @@ use crate::{ + defaults::{default_page, default_page_size}, model::PlaylistItemType, - utils::{default_page, default_page_size}, }; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; diff --git a/shared/src/utils/constants.rs b/shared/src/utils/constants.rs index 64b9aeed9..86e0c9ee5 100644 --- a/shared/src/utils/constants.rs +++ b/shared/src/utils/constants.rs @@ -1,3 +1,9 @@ +//! System-wide constants that aren't config defaults. +//! +//! Config-default constants (paths, sizes, suffixes, fallback filenames) live in +//! `crate::defaults::*` and are re-exported through `crate::utils::*`. + +use crate::defaults::DEFAULT_EPG_NORMALIZE_REGEX; use regex::Regex; use std::{ collections::HashSet, @@ -7,49 +13,11 @@ use std::{ pub const DEFAULT_HOME_ENV_VAR: &str = "TULIPROX_HOME"; pub const DEFAULT_WEB_ROOT_ENV_VAR: &str = "TULIPROX_WEB_ROOT"; -pub const DEFAULT_USER_AGENT: &str = "VLC/3.0.16 LibVLC/3.0.16"; -pub const DEFAULT_BACKUP_DIR: &str = "backup"; -pub const DEFAULT_CACHE_DIR: &str = "cache"; -pub const DEFAULT_STORAGE_TEMP_DIR: &str = "tmp"; -pub const DEFAULT_USER_CONFIG_DIR: &str = "user_config"; -pub const DEFAULT_DOWNLOAD_DIR: &str = "downloads"; -pub const DEFAULT_CUSTOM_STREAM_RESPONSE_PATH: &str = "resources"; pub const DNS_RESOLVED_FILE: &str = "provider_dns_resolved.json"; -pub const USER_FILE: &str = "user.txt"; -pub const USER_GROUP_FILE: &str = "groups.txt"; -pub const CONFIG_PATH: &str = "config"; -pub const CONFIG_FILE: &str = "config.yml"; -pub const SOURCE_FILE: &str = "source.yml"; -pub const MAPPING_FILE: &str = "mapping.yml"; -pub const TEMPLATE_FILE: &str = "template.yml"; -pub const API_PROXY_FILE: &str = "api-proxy.yml"; -pub const DEFAULT_WEB_DIR: &str = "web"; -pub const DEFAULT_STORAGE_DIR: &str = "data"; // TODO rename to storage and use data for config, storage, ... -pub const DEFAULT_PORT: u16 = 8901; + pub const ENCODING_GZIP: &str = "gzip"; pub const ENCODING_DEFLATE: &str = "deflate"; -pub const HLS_EXT: &str = ".m3u8"; -pub const DASH_EXT: &str = ".mpd"; - -pub const HLS_PREFIX: &str = "hls"; -pub const CUSTOM_VIDEO_PREFIX: &str = "cvs"; - -pub const HLS_EXT_QUERY: &str = ".m3u8?"; -pub const HLS_EXT_FRAGMENT: &str = ".m3u8#"; -pub const DASH_EXT_QUERY: &str = ".mpd?"; -pub const DASH_EXT_FRAGMENT: &str = ".mpd#"; - -pub const CHANNEL_UNAVAILABLE: &str = "channel_unavailable.ts"; -pub const USER_CONNECTIONS_EXHAUSTED: &str = "user_connections_exhausted.ts"; -pub const PROVIDER_CONNECTIONS_EXHAUSTED: &str = "provider_connections_exhausted.ts"; -pub const LOW_PRIORITY_PREEMPTED: &str = "low_priority_preempted.ts"; -pub const USER_ACCOUNT_EXPIRED: &str = "user_account_expired.ts"; -pub const PANEL_API_PROVISIONING: &str = "panel_api_provisioning.ts"; - -pub const DEFAULT_EPISODE_PATTERN: &str = r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*"; -pub const DEFAULT_EPG_NORMALIZE_REGEX: &str = r"[^a-zA-Z0-9\-]"; - pub const FILENAME_TRIM_PATTERNS: &[char] = &['.', '-', '_']; const SUPPORTED_RESPONSE_HEADERS: &[&str] = &[ diff --git a/shared/src/utils/default_utils.rs b/shared/src/utils/default_utils.rs deleted file mode 100644 index c9408b44e..000000000 --- a/shared/src/utils/default_utils.rs +++ /dev/null @@ -1,484 +0,0 @@ -use crate::{ - model::{ - ClusterFlags, ConfigTargetOptions, LibraryMetadataFormat, MetadataUpdateConfigDto, ProcessingOrder, - RuntimeConfigReportFormat, VideoConfigDto, - }, - utils::{ - CONFIG_PATH, DEFAULT_BACKUP_DIR, DEFAULT_CACHE_DIR, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH, DEFAULT_DOWNLOAD_DIR, - DEFAULT_EPG_NORMALIZE_REGEX, DEFAULT_EPISODE_PATTERN, DEFAULT_STORAGE_DIR, DEFAULT_USER_AGENT, - DEFAULT_USER_CONFIG_DIR, DEFAULT_WEB_DIR, MAPPING_FILE, TEMPLATE_FILE, USER_FILE, USER_GROUP_FILE, - }, -}; -use std::sync::Arc; - -pub const fn is_zero_u16(v: &u16) -> bool { *v == 0 } -pub const fn is_zero_i16(v: &i16) -> bool { *v == 0 } -pub const fn is_zero_u32(v: &u32) -> bool { *v == 0 } -pub const fn is_true(v: &bool) -> bool { *v } -pub const fn is_false(v: &bool) -> bool { !*v } -pub const fn default_as_true() -> bool { true } - -pub fn is_blank_optional_string(s: &Option) -> bool { - s.as_ref().is_none_or(|s| s.chars().all(|c| c.is_whitespace())) -} - -pub fn is_non_blank_optional_string(s: &Option) -> bool { !is_blank_optional_string(s) } - -pub fn is_blank_optional_arc_str(s: &Option>) -> bool { - s.as_ref().is_none_or(|s| s.chars().all(|c| c.is_whitespace())) -} - -pub fn is_empty_optional_vec(s: &Option>) -> bool { s.as_ref().is_none_or(|v| v.is_empty()) } - -pub fn default_as_default() -> String { "default".into() } - -pub fn default_page() -> u32 { 1 } - -pub fn default_page_size() -> u16 { 25 } - -pub const fn is_default_runtime_config_report_format(value: &RuntimeConfigReportFormat) -> bool { - matches!(value, RuntimeConfigReportFormat::Yaml) -} - -pub fn is_cluster_optional(cf: &Option) -> bool { cf.is_none_or(|c| c.is_all()) } - -macro_rules! default_eq_fns { - ($( $default_fn:ident, $is_default_fn:ident, $ty:ty, $value:expr; )* ) => { - $( - pub const fn $default_fn() -> $ty { $value } - pub const fn $is_default_fn(v: &$ty) -> bool { *v == $default_fn() } - )* - }; -} - -// Default delay values for resolving VOD or Series requests, -// used to prevent frequent requests that could trigger a provider ban. -// Default delay values for probing streams (ffprobe), -// used to avoid excessive probing under rapid playlist changes. -// Default grace values to accommodate rapid channel changes and seek requests, -// helping avoid triggering hard max_connection enforcement. -default_eq_fns!( - default_resolve_delay_secs, is_default_resolve_delay_secs, u16, 2; - default_probe_delay_secs, is_default_probe_delay_secs, u16, 2; - default_grace_period_millis, is_default_grace_period_millis, u64, 2000; - default_shared_burst_buffer_mb, is_default_shared_burst_buffer_mb, u64, 12; - default_grace_period_timeout_secs, is_default_grace_period_timeout_secs, u64, 4; - default_hls_session_ttl_secs, is_default_hls_session_ttl_secs, u64, 15; - default_catchup_session_ttl_secs, is_default_catchup_session_ttl_secs, u64, 45; - default_connect_timeout_secs, is_default_connect_timeout_secs, u32, 6; - default_resource_retry_attempts, is_default_resource_retry_attempts, u32, 3; - default_resource_retry_backoff_ms, is_default_resource_retry_backoff_ms, u64, 250; - default_interner_gc_interval_secs, is_default_interner_gc_interval_secs, u32, 180; - default_interner_gc_min_pool_size, is_default_interner_gc_min_pool_size, u32, 100; - default_custom_stream_response_error_status, is_default_custom_stream_response_error_status, u16, 502; -); -pub const fn default_panel_api_provision_timeout_secs() -> u64 { 65 } -pub const fn default_panel_api_provision_probe_interval_secs() -> u64 { 15 } -pub const fn default_panel_api_provision_cooldown_secs() -> u64 { 0 } -pub const fn default_panel_api_alias_pool_min() -> u16 { 1 } -pub const fn default_panel_api_alias_pool_max() -> u16 { 1 } - -//_---------- -pub const fn default_resource_retry_backoff_multiplier() -> f64 { 1.0 } -pub const F64_DEFAULT_EPSILON: f64 = 1e-9; -pub const fn is_default_resource_retry_backoff_multiplier(v: &f64) -> bool { - (*v - default_resource_retry_backoff_multiplier()).abs() < F64_DEFAULT_EPSILON -} - -fn fill_with_secure_random_bytes(out: &mut [u8]) { - #[cfg(target_arch = "wasm32")] - { - for byte in out { - *byte = fastrand::u8(..); - } - } - - #[cfg(not(target_arch = "wasm32"))] - if let Err(err) = getrandom::fill(out) { - panic!("failed to generate secure random bytes: {err}"); - } -} - -pub fn generate_default_access_secret() -> [u8; 32] { - let mut out = [0u8; 32]; - fill_with_secure_random_bytes(&mut out); - out -} - -pub fn generate_default_encrypt_secret() -> [u8; 16] { - let mut out = [0u8; 16]; - fill_with_secure_random_bytes(&mut out); - out -} - -pub fn default_secret() -> String { generate_default_encrypt_secret().iter().map(|b| format!("{:02X}", b)).collect() } - -// 30 minutes by default; `0` still means “no expiration.” -default_eq_fns!( - default_kick_secs, is_default_kick_secs, u64, 90; - default_token_ttl_mins, is_default_token_ttl_mins, u32, 30; - default_auth_error_status, is_default_auth_error_status, u16, 403; - default_epg_match_threshold, is_default_epg_match_threshold, u16, 80; - default_epg_best_match_threshold, is_default_epg_best_match_threshold, u16, 95; -); -pub fn default_epg_normalize_regex() -> Option { Some(DEFAULT_EPG_NORMALIZE_REGEX.to_string()) } -pub fn is_default_epg_normalize_regex(v: &Option) -> bool { - match v.as_ref().map(|value| value.trim()) { - None => true, - Some(value) => value.is_empty() || value == DEFAULT_EPG_NORMALIZE_REGEX, - } -} -pub const DEFAULT_EPG_STRIP: &[&str] = &["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw", "full hd"]; -pub const DEFAULT_EPG_NAME_PREFIX_SEPARATOR: &[char] = &[':', '|', '-']; -pub fn default_epg_strip() -> Option> { - Some(DEFAULT_EPG_STRIP.iter().map(|item| (*item).to_string()).collect()) -} -pub fn is_default_epg_strip(v: &Option>) -> bool { - let Some(current) = v.as_ref() else { - return true; - }; - let Some(default_strip) = default_epg_strip() else { - return false; - }; - current == &default_strip -} -pub fn default_epg_name_prefix_separator() -> Option> { Some(DEFAULT_EPG_NAME_PREFIX_SEPARATOR.to_vec()) } -pub fn is_default_epg_name_prefix_separator(v: &Option>) -> bool { - let Some(current) = v.as_ref() else { - return true; - }; - let Some(default_separator) = default_epg_name_prefix_separator() else { - return false; - }; - current == &default_separator -} - -pub const fn default_tmdb_match_threshold() -> u16 { 86 } -pub const fn is_default_tmdb_match_threshold(v: &u16) -> bool { *v == default_tmdb_match_threshold() } - -pub const TMDB_API_KEY: &str = "4219e299c89411838049ab0dab19ebd5"; -pub fn default_tmdb_api_key() -> Option { Some(TMDB_API_KEY.to_string()) } -pub fn is_tmdb_default_api_key(s: &Option) -> bool { s.as_ref().is_none_or(|s| s == TMDB_API_KEY) } -pub fn is_default_tmdb_language(v: &String) -> bool { v == DEFAULT_TMDB_LANGUAGE } - -pub const DEFAULT_METADATA_PATH: &str = "metadata"; -pub fn default_metadata_path() -> String { DEFAULT_METADATA_PATH.to_string() } -pub fn is_default_metadata_path(s: &str) -> bool { s == DEFAULT_METADATA_PATH } - -pub const DEFAULT_TMDB_RATE_LIMIT_MS: u64 = 250; -pub const DEFAULT_TMDB_CACHE_DURATION_DAYS: u32 = 30; -pub const DEFAULT_TMDB_LANGUAGE: &str = "en-US"; -pub const fn default_tmdb_rate_limit_ms() -> u64 { DEFAULT_TMDB_RATE_LIMIT_MS } -pub const fn default_tmdb_cache_duration_days() -> u32 { DEFAULT_TMDB_CACHE_DURATION_DAYS } -pub fn default_tmdb_language() -> String { DEFAULT_TMDB_LANGUAGE.to_owned() } -pub fn is_default_tmdb_rate_limit_ms(v: &u64) -> bool { *v == DEFAULT_TMDB_RATE_LIMIT_MS } -pub fn is_default_tmdb_cache_duration_days(v: &u32) -> bool { *v == DEFAULT_TMDB_CACHE_DURATION_DAYS } - -pub fn default_storage_formats() -> Vec { vec![] } -pub fn default_movie_category() -> String { String::from("Local Movies") } -pub fn default_series_category() -> String { String::from("Local TV Shows") } - -pub fn default_thumbnail_width() -> u32 { 320 } -pub fn default_thumbnail_height() -> u32 { 180 } -pub fn default_thumbnail_quality() -> u8 { 75 } - -pub const DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS: &[&str] = &["mp4", "mkv", "avi", "mov", "ts", "m4v", "webm"]; - -pub fn default_supported_library_extensions() -> Vec { - DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS.iter().map(|s| (*s).to_owned()).collect() -} - -pub fn is_default_supported_library_extensions(v: &[String]) -> bool { - v.len() == DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS.len() - && v.iter().zip(DEFAULT_SUPPORTED_LIBRARY_EXTENSIONS).all(|(a, b)| a == b) -} - -pub const DEFAULT_VIDEO_EXTENSIONS: &[&str] = &["mkv", "avi", "mp4", "mpeg", "divx", "mov"]; - -pub fn default_supported_video_extensions() -> Vec { - DEFAULT_VIDEO_EXTENSIONS.iter().map(|s| (*s).to_owned()).collect() -} - -pub fn is_default_supported_video_extensions(v: &[String]) -> bool { - v.len() == DEFAULT_VIDEO_EXTENSIONS.len() && v.iter().zip(DEFAULT_VIDEO_EXTENSIONS).all(|(a, b)| a == b) -} - -pub fn is_config_target_options_empty(v: &Option) -> bool { - v.as_ref().is_none_or(|c| c.is_empty()) -} - -pub fn is_default_processing_order(p: &ProcessingOrder) -> bool { *p == ProcessingOrder::default() } - -pub const fn default_probe_live_interval() -> u32 { 120 } - -pub const fn is_default_probe_live_interval(v: &u32) -> bool { *v == default_probe_live_interval() } - -pub const fn default_resolve_background() -> bool { true } -pub const fn default_xtream_live_stream_use_prefix() -> bool { true } - -pub fn default_metadata_queue_log_interval() -> String { "30s".to_string() } -pub fn is_default_metadata_queue_log_interval(v: &String) -> bool { *v == default_metadata_queue_log_interval() } -pub fn default_metadata_progress_log_interval() -> String { "15s".to_string() } -pub fn is_default_metadata_progress_log_interval(v: &String) -> bool { *v == default_metadata_progress_log_interval() } -pub fn default_metadata_max_resolve_retry_backoff() -> String { "1h".to_string() } -pub fn is_default_metadata_max_resolve_retry_backoff(v: &String) -> bool { - *v == default_metadata_max_resolve_retry_backoff() -} -pub fn default_metadata_resolve_min_retry_base() -> String { "5s".to_string() } -pub fn is_default_metadata_resolve_min_retry_base(v: &String) -> bool { - *v == default_metadata_resolve_min_retry_base() -} -pub fn default_metadata_resolve_exhaustion_reset_gap() -> String { "1h".to_string() } -pub fn is_default_metadata_resolve_exhaustion_reset_gap(v: &String) -> bool { - *v == default_metadata_resolve_exhaustion_reset_gap() -} -pub fn default_metadata_probe_cooldown() -> String { "7d".to_string() } -pub fn is_default_metadata_probe_cooldown(v: &String) -> bool { *v == default_metadata_probe_cooldown() } -pub fn default_metadata_tmdb_cooldown() -> String { "7d".to_string() } -pub fn is_default_metadata_tmdb_cooldown(v: &String) -> bool { *v == default_metadata_tmdb_cooldown() } -pub fn default_metadata_retry_delay() -> String { "2s".to_string() } -pub fn is_default_metadata_retry_delay(v: &String) -> bool { *v == default_metadata_retry_delay() } -pub fn default_metadata_probe_retry_load_retry_delay() -> String { "1m".to_string() } -pub fn is_default_metadata_probe_retry_load_retry_delay(v: &String) -> bool { - *v == default_metadata_probe_retry_load_retry_delay() -} -pub fn default_metadata_worker_idle_timeout() -> String { "1m".to_string() } -pub fn is_default_metadata_worker_idle_timeout(v: &String) -> bool { *v == default_metadata_worker_idle_timeout() } -pub fn default_metadata_probe_retry_backoff_step_1() -> String { "10m".to_string() } -pub fn is_default_metadata_probe_retry_backoff_step_1(v: &String) -> bool { - *v == default_metadata_probe_retry_backoff_step_1() -} -pub fn default_metadata_probe_retry_backoff_step_2() -> String { "30m".to_string() } -pub fn is_default_metadata_probe_retry_backoff_step_2(v: &String) -> bool { - *v == default_metadata_probe_retry_backoff_step_2() -} -pub fn default_metadata_probe_retry_backoff_step_3() -> String { "1h".to_string() } -pub fn is_default_metadata_probe_retry_backoff_step_3(v: &String) -> bool { - *v == default_metadata_probe_retry_backoff_step_3() -} -pub const fn default_metadata_max_attempts_resolve() -> u8 { 3 } -pub const fn is_default_metadata_max_attempts_resolve(v: &u8) -> bool { *v == default_metadata_max_attempts_resolve() } -pub const fn default_metadata_max_attempts_probe() -> u8 { 3 } -pub const fn is_default_metadata_max_attempts_probe(v: &u8) -> bool { *v == default_metadata_max_attempts_probe() } -pub const fn default_metadata_backoff_jitter_percent() -> u8 { 20 } -pub const fn is_default_metadata_backoff_jitter_percent(v: &u8) -> bool { - *v == default_metadata_backoff_jitter_percent() -} -pub const fn default_metadata_max_queue_size() -> usize { 100_000 } -pub const fn is_default_metadata_max_queue_size(v: &usize) -> bool { *v == default_metadata_max_queue_size() } -pub const fn default_metadata_no_change_cache_ttl_secs() -> u64 { 3600 } -pub const fn is_default_metadata_no_change_cache_ttl_secs(v: &u64) -> bool { - *v == default_metadata_no_change_cache_ttl_secs() -} -pub const fn default_metadata_probe_fairness_resolve_burst() -> usize { 200 } -pub const fn is_default_metadata_probe_fairness_resolve_burst(v: &usize) -> bool { - *v == default_metadata_probe_fairness_resolve_burst() -} -pub fn default_metadata_ffprobe_analyze_duration() -> String { "10s".to_string() } -pub fn is_default_metadata_ffprobe_analyze_duration(v: &String) -> bool { - *v == default_metadata_ffprobe_analyze_duration() -} -pub fn default_metadata_ffprobe_probe_size() -> String { "10MB".to_string() } -pub fn is_default_metadata_ffprobe_probe_size(v: &String) -> bool { *v == default_metadata_ffprobe_probe_size() } -pub fn default_metadata_ffprobe_live_analyze_duration() -> String { "5s".to_string() } -pub fn is_default_metadata_ffprobe_live_analyze_duration(v: &String) -> bool { - *v == default_metadata_ffprobe_live_analyze_duration() -} -pub fn default_metadata_ffprobe_live_probe_size() -> String { "5MB".to_string() } -pub fn is_default_metadata_ffprobe_live_probe_size(v: &String) -> bool { - *v == default_metadata_ffprobe_live_probe_size() -} -pub fn default_probe_user_priority() -> i8 { 127 } -pub fn is_default_probe_user_priority(v: &i8) -> bool { *v == default_probe_user_priority() } -pub fn default_user_priority() -> i8 { 0 } -pub fn is_default_user_priority(v: &i8) -> bool { *v == default_user_priority() } - -pub fn get_default_web_root() -> String { DEFAULT_WEB_DIR.to_string() } -pub fn is_blank_or_default_web_root(value: &str) -> bool { - let normalized = value.trim().replace('\\', "/"); - if normalized.is_empty() { - return true; - } - - let normalized = normalized.trim_end_matches('/'); - normalized.trim_start_matches("./") == DEFAULT_WEB_DIR -} - -pub fn is_default_dir_path(value: &str, default_dir: &str) -> bool { - let normalized = value.trim().replace('\\', "/"); - let normalized = normalized.trim_end_matches('/'); - let normalized = normalized.trim_start_matches("./"); - let default_dir = default_dir.trim().replace('\\', "/"); - let default_dir = default_dir.trim_end_matches('/'); - normalized == default_dir -} - -pub fn is_blank_or_default_download_dir(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_DOWNLOAD_DIR)) -} -pub fn default_download_dir() -> Option { Some(DEFAULT_DOWNLOAD_DIR.to_string()) } - -pub fn default_episode_pattern() -> Option { Some(DEFAULT_EPISODE_PATTERN.to_string()) } - -pub fn is_blank_or_default_episode_pattern(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || value.trim() == DEFAULT_EPISODE_PATTERN) -} - -pub fn is_blank_or_default_cache_dir(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_CACHE_DIR)) -} - -pub fn default_default_user_agent() -> Option { Some(DEFAULT_USER_AGENT.to_string()) } -pub fn default_main_storage_dir() -> Option { Some(DEFAULT_STORAGE_DIR.to_string()) } -pub fn default_main_backup_dir() -> Option { Some(DEFAULT_BACKUP_DIR.to_string()) } -pub fn default_main_user_config_dir() -> Option { Some(DEFAULT_USER_CONFIG_DIR.to_string()) } -pub fn default_main_mapping_path() -> Option { Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}")) } -pub fn default_main_template_path() -> Option { Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}")) } -pub fn default_custom_stream_response_path() -> Option { Some(DEFAULT_CUSTOM_STREAM_RESPONSE_PATH.to_string()) } -pub fn default_user_file_path() -> Option { Some(format!("./{CONFIG_PATH}/{USER_FILE}")) } - -pub fn default_user_group_file_path() -> Option { Some(format!("./{CONFIG_PATH}/{USER_GROUP_FILE}")) } - -fn is_default_config_file_path(value: &str, file_name: &str) -> bool { - let normalized = value.trim().replace('\\', "/"); - let normalized = normalized.trim_start_matches("./"); - normalized == file_name - || normalized.rsplit_once('/').is_some_and(|(dir, file)| dir == CONFIG_PATH && file == file_name) -} - -pub fn is_blank_or_default_custom_stream_response_path(path: &Option) -> bool { - path.as_ref() - .is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH)) -} - -pub fn is_blank_or_default_mapping_path(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, MAPPING_FILE)) -} - -pub fn is_blank_or_default_template_path(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, TEMPLATE_FILE)) -} - -pub fn is_blank_or_default_storage_dir(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_STORAGE_DIR)) -} - -pub fn is_blank_or_default_backup_dir(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_BACKUP_DIR)) -} - -pub fn is_blank_or_default_user_config_dir(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_dir_path(value, DEFAULT_USER_CONFIG_DIR)) -} - -pub fn is_blank_or_default_user_file_path(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, USER_FILE)) -} - -pub fn is_blank_or_default_user_group_file_path(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || is_default_config_file_path(value, USER_GROUP_FILE)) -} - -pub fn normalize_optional_dir(path: &Option, default_dir: &str) -> Option { - path.as_ref().and_then(|value| { - if value.trim().is_empty() || is_default_dir_path(value, default_dir) { - None - } else { - Some(value.clone()) - } - }) -} - -pub fn normalize_optional_config_file_path(path: &Option, default_file_name: &str) -> Option { - path.as_ref().and_then(|value| { - if value.trim().is_empty() || is_default_config_file_path(value, default_file_name) { - None - } else { - Some(value.clone()) - } - }) -} - -pub fn is_none_or_empty_video(video: &Option) -> bool { - video.as_ref().is_none_or(VideoConfigDto::is_empty) -} -pub fn is_none_or_empty_metadata_update(metadata_update: &Option) -> bool { - metadata_update.as_ref().is_none_or(MetadataUpdateConfigDto::is_empty) -} - -////////////////////////////// -// HDHomerun Device Defaults -////////////////////////////// -const DEFAULT_FRIENDLY_NAME: &str = "TuliproxTV"; -const DEFAULT_MANUFACTURER: &str = "Silicondust"; -const DEFAULT_MODEL_NAME: &str = "HDTC-2US"; -const DEFAULT_FIRMWARE_NAME: &str = "hdhomeruntc_atsc"; -const DEFAULT_FIRMWARE_VERSION: &str = "20170930"; -const DEFAULT_DEVICE_TYPE: &str = "urn:schemas-upnp-org:device:MediaServer:1"; -const DEFAULT_DEVICE_UDN: &str = - "uuid:12345678-90ab-cdef-1234-567890abcdef::urn:dial-multicast:com.silicondust.hdhomerun"; -pub fn default_friendly_name() -> String { DEFAULT_FRIENDLY_NAME.into() } -pub fn default_manufacturer() -> String { DEFAULT_MANUFACTURER.into() } -pub fn default_model_name() -> String { DEFAULT_MODEL_NAME.into() } -pub fn default_firmware_name() -> String { DEFAULT_FIRMWARE_NAME.into() } -pub fn default_firmware_version() -> String { DEFAULT_FIRMWARE_VERSION.into() } -pub fn default_device_type() -> String { DEFAULT_DEVICE_TYPE.into() } -pub fn default_device_udn() -> String { DEFAULT_DEVICE_UDN.into() } -pub fn is_default_friendly_name(value: &String) -> bool { value == DEFAULT_FRIENDLY_NAME } -pub fn is_default_manufacturer(value: &String) -> bool { value == DEFAULT_MANUFACTURER } -pub fn is_default_model_name(value: &String) -> bool { value == DEFAULT_MODEL_NAME } -pub fn is_default_firmware_name(value: &String) -> bool { value == DEFAULT_FIRMWARE_NAME } -pub fn is_default_firmware_version(value: &String) -> bool { value == DEFAULT_FIRMWARE_VERSION } -pub fn is_default_device_type(value: &String) -> bool { value == DEFAULT_DEVICE_TYPE } -pub fn is_default_device_udn(value: &String) -> bool { value == DEFAULT_DEVICE_UDN } - -////////////////////////// -// trakt -//////////////////////////// -pub const TRAKT_API_KEY: &str = "0183a05ad97098d87287fe46da4ae286f434f32e8e951caad4cc147c947d79a3"; -pub const TRAKT_API_VERSION: &str = "2"; -pub const TRAKT_API_URL: &str = "https://api.trakt.tv"; - -pub fn default_trakt_api_key() -> String { String::from(TRAKT_API_KEY) } - -pub fn default_trakt_api_version() -> String { String::from(TRAKT_API_VERSION) } - -pub fn default_trakt_api_url() -> String { String::from(TRAKT_API_URL) } - -pub fn default_trakt_fuzzy_threshold() -> u8 { 80 } - -///////////////////////////////////// -// Stream History -//////////////////////////////////// -pub const DEFAULT_STREAM_HISTORY_BATCH_SIZE: usize = 128; -pub const DEFAULT_STREAM_HISTORY_RETENTION_DAYS: u16 = 30; -pub const DEFAULT_STREAM_HISTORY_DIR: &str = "stream_history"; - -pub fn default_stream_history_batch_size() -> usize { DEFAULT_STREAM_HISTORY_BATCH_SIZE } - -pub fn default_stream_history_retention_days() -> u16 { DEFAULT_STREAM_HISTORY_RETENTION_DAYS } - -pub fn default_stream_history_directory() -> String { DEFAULT_STREAM_HISTORY_DIR.to_string() } - -pub fn is_default_stream_history_batch_size(batch_size: &usize) -> bool { - *batch_size == DEFAULT_STREAM_HISTORY_BATCH_SIZE -} - -pub fn is_default_stream_history_retention_days(retention_days: &u16) -> bool { - *retention_days == DEFAULT_STREAM_HISTORY_RETENTION_DAYS -} - -pub fn is_blank_stream_history_directory(directory: &str) -> bool { directory.trim().is_empty() } - -pub const fn default_media_server_catalog_page_size() -> u16 { 100 } -pub const fn default_media_server_catalog_request_delay_ms() -> u64 { 250 } -pub const fn is_default_media_server_catalog_page_size(value: &u16) -> bool { - *value == default_media_server_catalog_page_size() -} -pub const fn is_default_media_server_catalog_request_delay_ms(value: &u64) -> bool { - *value == default_media_server_catalog_request_delay_ms() -} - -pub fn default_warn_percent() -> f64 { 80.0 } -pub fn default_critical_percent() -> f64 { 95.0 } -pub fn default_repeat_interval_secs() -> u64 { 3600 } diff --git a/shared/src/utils/mod.rs b/shared/src/utils/mod.rs index 306dac1b0..43885ff8e 100644 --- a/shared/src/utils/mod.rs +++ b/shared/src/utils/mod.rs @@ -1,7 +1,5 @@ mod bitset; -mod constants; mod crypto; -mod default_utils; mod directed_graph; pub mod flags; mod hash_utils; @@ -16,10 +14,11 @@ mod string_interner; mod string_utils; mod time_utils; +pub mod constants; + pub use self::{ constants::*, crypto::*, - default_utils::*, directed_graph::*, flags::{country_code_to_index, index_to_country_code, FlagEntry, FlagsLoader, DEFAULT_COMPRESSION_LEVEL}, hash_utils::*, diff --git a/shared/src/utils/request.rs b/shared/src/utils/request.rs index 89e0aa9b2..0ddccd6f7 100644 --- a/shared/src/utils/request.rs +++ b/shared/src/utils/request.rs @@ -1,6 +1,7 @@ use crate::{ + defaults::{DASH_EXT, DASH_EXT_FRAGMENT, DASH_EXT_QUERY, HLS_EXT, HLS_EXT_FRAGMENT, HLS_EXT_QUERY}, error::TuliproxError, - utils::{CONSTANTS, DASH_EXT, DASH_EXT_FRAGMENT, DASH_EXT_QUERY, HLS_EXT, HLS_EXT_FRAGMENT, HLS_EXT_QUERY}, + utils::CONSTANTS, }; use std::{borrow::Cow, sync::atomic::Ordering}; use url::Url; diff --git a/shared/src/utils/size_utils.rs b/shared/src/utils/size_utils.rs index 7d3fa307f..6b18438b4 100644 --- a/shared/src/utils/size_utils.rs +++ b/shared/src/utils/size_utils.rs @@ -5,11 +5,15 @@ pub const fn bytes_to_megabytes(bytes: u64) -> u64 { bytes / 1_048_576 } pub fn parse_size_base_2(size_str: &str) -> Result { let units = [ - ("KB", 1_024u64), // Kilobytes - ("MB", 1_048_576u64), // Megabytes - ("GB", 1_073_741_824u64), // Gigabytes - ("TB", 1_099_511_628_000u64), // Terabytes - ("B", 1u64), // Bytes + ("TB", 1_099_511_627_776u64), // Terabytes + ("TIB", 1_099_511_627_776u64), // Tebibytes (alias, same multiplier) + ("GB", 1_073_741_824u64), // Gigabytes + ("GIB", 1_073_741_824u64), // Gibibytes (alias) + ("MB", 1_048_576u64), // Megabytes + ("MIB", 1_048_576u64), // Mebibytes (alias) + ("KB", 1_024u64), // Kilobytes + ("KIB", 1_024u64), // Kibibytes (alias) + ("B", 1u64), // Bytes ]; let size_str = size_str.trim().to_uppercase(); @@ -101,7 +105,7 @@ pub fn human_readable_kbps(kbps: u64) -> String { #[cfg(test)] mod tests { - use crate::utils::parse_to_kbps; + use crate::utils::{parse_size_base_2, parse_to_kbps}; #[test] fn test_parse_kpbs() { @@ -115,4 +119,13 @@ mod tests { assert_eq!(parse_to_kbps("1Mbps").unwrap(), 1000); assert_eq!(parse_to_kbps("1Mibps").unwrap(), 1024); } + + #[test] + fn test_parse_size_base_2_accepts_kib_mib_gib_aliases() { + assert_eq!(parse_size_base_2("1KiB").unwrap(), 1024); + assert_eq!(parse_size_base_2("1MiB").unwrap(), 1024 * 1024); + assert_eq!(parse_size_base_2("1GiB").unwrap(), 1024_u64.pow(3)); + assert_eq!(parse_size_base_2("1TiB").unwrap(), 1024_u64.pow(4)); + assert_eq!(parse_size_base_2("2gib").unwrap(), 2 * 1024_u64.pow(3)); // case-insensitive + } } diff --git a/shared/src/utils/string_utils.rs b/shared/src/utils/string_utils.rs index 3d405d749..b3262a2ca 100644 --- a/shared/src/utils/string_utils.rs +++ b/shared/src/utils/string_utils.rs @@ -64,6 +64,16 @@ pub fn is_blank_optional_str(s: Option<&str>) -> bool { s.as_ref().is_none_or(|s| s.chars().all(|c| c.is_whitespace())) } +pub fn is_blank_optional_string(s: &Option) -> bool { + s.as_ref().is_none_or(|s| s.chars().all(|c| c.is_whitespace())) +} + +pub fn is_non_blank_optional_string(s: &Option) -> bool { !is_blank_optional_string(s) } + +pub fn is_blank_optional_arc_str(s: &Option<::std::sync::Arc>) -> bool { + s.as_ref().is_none_or(|s| s.chars().all(|c| c.is_whitespace())) +} + pub fn trim_slash(s: &str) -> Cow<'_, str> { let trimmed = s.trim_matches('/'); if trimmed.len() == s.len() {