diff --git a/.gitignore b/.gitignore index 2e402de12..71e9d3097 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,8 @@ backend/app/Cargo.lock # only exist on a developer's own machine), so it is not project source. /.claude/ /.mcp.json +AGENTS.md +GOAL.md # Local refactoring measurements (clippy inventories, cargo metadata, timings). # These are engineering artifacts of the modularization work, not project docs. diff --git a/Cargo.lock b/Cargo.lock index 9593ae8b3..9528d7778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,7 +1292,7 @@ checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb" [[package]] name = "frontend" -version = "3.3.155" +version = "3.3.156" dependencies = [ "base64 0.23.1", "blake3", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "shared" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "base64 0.23.1", @@ -4399,7 +4399,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tuliprox" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "arc-swap", @@ -4453,6 +4453,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-stream", + "tokio-tungstenite", "tokio-util", "tower", "tower-http", @@ -4479,7 +4480,7 @@ dependencies = [ [[package]] name = "tuliprox-auth" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "axum", @@ -4502,7 +4503,7 @@ dependencies = [ [[package]] name = "tuliprox-btree" -version = "3.3.155" +version = "3.3.156" dependencies = [ "blake3", "crc32fast", @@ -4522,7 +4523,7 @@ dependencies = [ [[package]] name = "tuliprox-config-loader" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "blake3", @@ -4542,7 +4543,7 @@ dependencies = [ [[package]] name = "tuliprox-core" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "arc-swap", @@ -4593,7 +4594,7 @@ dependencies = [ [[package]] name = "tuliprox-curation" -version = "3.3.155" +version = "3.3.156" dependencies = [ "indexmap", "log", @@ -4610,7 +4611,7 @@ dependencies = [ [[package]] name = "tuliprox-dvr" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "blake3", @@ -4638,7 +4639,7 @@ dependencies = [ [[package]] name = "tuliprox-hls" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "async-compression", @@ -4675,7 +4676,7 @@ dependencies = [ [[package]] name = "tuliprox-iptv" -version = "3.3.155" +version = "3.3.156" dependencies = [ "base64 0.23.1", "bytes", @@ -4700,7 +4701,7 @@ dependencies = [ [[package]] name = "tuliprox-library" -version = "3.3.155" +version = "3.3.156" dependencies = [ "blake3", "bytes", @@ -4727,7 +4728,7 @@ dependencies = [ [[package]] name = "tuliprox-media-server" -version = "3.3.155" +version = "3.3.156" dependencies = [ "bytes", "futures", @@ -4746,7 +4747,7 @@ dependencies = [ [[package]] name = "tuliprox-messaging" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4768,7 +4769,7 @@ dependencies = [ [[package]] name = "tuliprox-metadata" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4791,7 +4792,7 @@ dependencies = [ [[package]] name = "tuliprox-mpegts" -version = "3.3.155" +version = "3.3.156" dependencies = [ "aes", "bytes", @@ -4805,7 +4806,7 @@ dependencies = [ [[package]] name = "tuliprox-parser" -version = "3.3.155" +version = "3.3.156" dependencies = [ "chrono", "chrono-tz", @@ -4825,7 +4826,7 @@ dependencies = [ [[package]] name = "tuliprox-processing" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "chrono", @@ -4864,7 +4865,7 @@ dependencies = [ [[package]] name = "tuliprox-repository" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "base64 0.23.1", @@ -4902,7 +4903,7 @@ dependencies = [ [[package]] name = "tuliprox-session" -version = "3.3.155" +version = "3.3.156" dependencies = [ "arc-swap", "axum", @@ -4932,7 +4933,7 @@ dependencies = [ [[package]] name = "tuliprox-testkit" -version = "3.3.155" +version = "3.3.156" dependencies = [ "axum", "base64 0.23.1", diff --git a/Cargo.toml b/Cargo.toml index 78ed20cee..cb44676c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"] resolver = "2" [workspace.package] -version = "3.3.155" +version = "3.3.156" edition = "2021" rust-version = "1.95.0" @@ -24,14 +24,6 @@ return_self_not_must_use = "allow" missing_errors_doc = "allow" large_futures = "allow" -# Its suggested rewrite of `assert!(x.is_empty())` is -# `assert_eq!(x, [] as [some::crate_root::Path; 0])`, which names a type that -# is usually not in scope at the assertion — which is why `clippy --fix` -# declines to apply it. The panic message improves; every one of the 39 call -# sites gets materially harder to read. Reverse this if the suggestion is -# ever made to use an in-scope type. -assert_is_empty = "allow" - [workspace.dependencies] shared = { version = "3", path = "shared" } tuliprox-hls = { version = "3", path = "backend/hls" } diff --git a/backend/app/Cargo.toml b/backend/app/Cargo.toml index f807aab62..cc5f2cb69 100644 --- a/backend/app/Cargo.toml +++ b/backend/app/Cargo.toml @@ -100,6 +100,7 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora vergen = { version = "10.0.3", features = ["build"] } [dev-dependencies] +tokio-tungstenite = "0.29.0" tuliprox-processing = { workspace = true, features = ["test-support"] } tuliprox-hls = { workspace = true, features = ["test-support"] } # The migration and playlist-stream tests build legacy v2 databases and diff --git a/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs index 65c189438..f592dbfed 100644 --- a/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs +++ b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs @@ -195,7 +195,7 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi .await; assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -326,7 +326,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi let admission = result.admission; assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -458,7 +458,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -641,7 +641,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same app_state.connection_manager.release_connection(&hls_addr).await; assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); let mut close_rx = app_state.connection_manager.get_close_connection_channel(); let result = resolve_admission_with_strategies( @@ -664,7 +664,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same assert_eq!(admission.permission(), UserConnectionPermission::Allowed); assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); assert_eq!( tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok), Some(crate::api::model::CloseConnectionSignal::WithReason( diff --git a/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs index 67f602c30..a9fcceed7 100644 --- a/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs +++ b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs @@ -128,7 +128,7 @@ async fn create_stream_response_details_preserves_stored_headers_when_fallback_o socket_bound: false, }) .await; - assert!(!created.is_empty()); + assert_ne!(created.len(), 0); app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; let busy = app_state.active_provider.acquire_exact_connection_with_grace( @@ -233,7 +233,7 @@ async fn force_provider_stream_response_clears_stored_headers_after_fallback_ope socket_bound: false, }) .await; - assert!(!created.is_empty()); + assert_ne!(created.len(), 0); app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; let busy = app_state.active_provider.acquire_exact_connection_with_grace( diff --git a/backend/app/src/api/config_file.rs b/backend/app/src/api/config_file.rs index 076159581..7f11410a6 100644 --- a/backend/app/src/api/config_file.rs +++ b/backend/app/src/api/config_file.rs @@ -777,7 +777,7 @@ mod tests { let refreshed = refresh_forced_targets(Arc::clone(¤t), &sources_with(vec![target(1, "my-target")])); assert!(!refreshed.enabled, "a reload must not enable a filter the operator never asked for"); - assert!(refreshed.target_names.is_empty()); + assert_eq!(refreshed.target_names.len(), 0); } /// Target ids are positional, so a surviving name must come back with @@ -808,6 +808,6 @@ mod tests { let refreshed = refresh_forced_targets(forced(&["gone"]), &sources_with(vec![target(1, "other")])); assert!(!refreshed.enabled, "an empty filter must not be left enabled: it would process nothing"); - assert!(refreshed.target_names.is_empty()); + assert_eq!(refreshed.target_names.len(), 0); } } diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index e59fd9bfe..32e4be8cb 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -21,7 +21,7 @@ use serde::Deserialize; use std::{str::FromStr, sync::Arc}; use tuliprox_hls::api::{ finite_hls_immutable_media_response, resolve_hls_standalone_custom_segment, HlsAccessLeaseId, - HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, + HlsMediaActivityCommitOutcome, HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, }; use url::form_urlencoded; @@ -286,7 +286,9 @@ async fn cvs_standalone_hls_segment_response( asset_fingerprint, index, now_ms, - ) { + ) + .await + { Ok(access) => access, Err( HlsStandaloneCustomSegmentError::InvalidIndex @@ -299,13 +301,28 @@ async fn cvs_standalone_hls_segment_response( { return *response; } - finite_hls_immutable_media_response( - access.bytes, + let response = finite_hls_immutable_media_response( + access.bytes.clone(), headers.get(header::RANGE), HLS_CVS_CONTENT_TYPE, HLS_CVS_IMMUTABLE_CACHE_CONTROL, method == Method::HEAD, - ) + ); + if response.status().is_success() { + match app_state + .hls + .proxy + .record_standalone_custom_segment_activity(&access_lease, &access, current_time_millis()) + .await + { + HlsMediaActivityCommitOutcome::Committed => {} + HlsMediaActivityCommitOutcome::StaleLeaseIdentity => return StatusCode::NOT_FOUND.into_response(), + HlsMediaActivityCommitOutcome::DeferredLockContention => { + return super::hls_terminal_response::hls_temporary_resource_unavailable_response(1_000) + } + } + } + response } async fn validate_hls_standalone_custom_access( diff --git a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs index c0b8c946d..f1eb75302 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs @@ -1,4 +1,5 @@ use super::{ + super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, clear_hls_provisioning_handoff_consumer, duration_to_millis_saturating, effective_hls_url_failover_provider_for_fetch_url, hls_access_lease_timing_for_session, hls_access_lease_ttl_ms, hls_availability_reevaluation_registration_failure_response, hls_cache_configured, @@ -29,8 +30,7 @@ use tuliprox_hls::api::{ safe_proxy_session_id, safe_session_key, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccountBindingProtection, HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, HlsOriginIoContext, HlsOriginWorkClass, - HlsPostRefreshRuntime, HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, - OriginRefreshRequest, ProxySessionId, RetryPolicy, + HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, ProxySessionId, }; #[allow(clippy::too_many_arguments)] @@ -251,13 +251,10 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest prepared_origin.runtime_provider_config.clone(), )?; { - let mut session_guard = session.write().await; + let session_guard = session.read().await; if session_guard.is_gc_marked_for_removal() { return Some(hls_canonical_retry_after_response()); } - if prepared_origin.origin_account_binding_to_store.is_some() { - session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store); - } } mark_hls_authorized_manifest_access(app_state, &session, now_ms).await; let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else( @@ -298,7 +295,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest (HlsManifestAcceptanceDirective::none(), Some(owner_key)) } HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => { - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + drop(prepared_origin.preacquired_origin_account_handle); return Some(hls_canonical_retry_after_response()); } }; @@ -317,7 +314,6 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest }; let origin_policy = hls_effective_origin_acquire_policy(&session).await; - let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle; let mut origin_io = HlsOriginIoContext { ctx: hls_ctx.clone(), @@ -328,49 +324,37 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest reservation_ttl_secs, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), }; if availability_reevaluation_owner_key.is_none() { if let Some(provider_handle) = preacquired_provider_handle.take() { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + origin_io = origin_io.with_managed_provider_handle(provider_handle); } } - let refresh_request = OriginRefreshRequest { - app_config: Arc::clone(&app_state.app_config), - session: Arc::clone(&session), - origin_entry, - headers, - origin_provider_session_headers, - client: app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: app_state.should_use_manual_redirects(), - segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), - hls_proxy: Arc::clone(&app_state.hls.proxy), - segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), - segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), - map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), - origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), - manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), - strip: app_state.hls.proxy.strip().clone(), - retry_policy: RetryPolicy::default(), - reverse_proxy_rewrite_secret: rewrite_secret.to_vec(), - transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), - manifest_commit_requirement, - fresh_manifest_requirement_generation: None, - acceptance_directive, - access_lease_id: Some(access_lease_id.clone()), - disabled_headers: app_state.get_disabled_headers(), - now_ms, - origin_io: Some(origin_io), - post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }), - }; + let refresh_request = hls_origin_refresh_request( + app_state, + &session, + tuliprox_hls::api::HlsPreparedOriginAccount { entry: origin_entry, snapshot: prepared_origin.snapshot }, + HlsOriginRefreshOptions { + headers, + origin_io, + manifest_commit_requirement, + acceptance_directive, + access_lease_id: Some(access_lease_id.clone()), + rewrite_secret: rewrite_secret.to_vec(), + now_ms, + }, + ); let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused { refresh_ordering } else { HlsManifestRefreshOrdering::Background }; if let Some(owner_key) = availability_reevaluation_owner_key { - app_state.connection_manager.release_provider_handle(preacquired_provider_handle); + drop(preacquired_provider_handle); touch_initial_manifest_access_lease_window( app_state, access_lease_id, diff --git a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs index 4864df106..4232efed7 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs @@ -217,7 +217,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); } match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => { + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => { let Some(response) = hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id) else { diff --git a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs index 70545f1c6..dfd74c712 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs @@ -516,6 +516,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_plans( pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry( plan: SharedHlsProvisioningSegmentPlan, content_length: u64, + media_start_ms: u64, duration_ms: u64, now_ms: u64, ) -> SegmentEntry { @@ -531,6 +532,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry( host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX), }, proxy_seq: plan.proxy_seq, + media_start_ms, duration_ms, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -636,10 +638,13 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); } session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + let media_start_ms = session_guard.next_media_start_ms; + session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms); session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); - session_guard - .segments - .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + session_guard.segments.insert( + plan.proxy_seq, + shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms), + ); inserted = true; } if inserted { @@ -682,10 +687,12 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); } session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + let media_start_ms = session_guard.next_media_start_ms; + session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms); session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); session_guard.segments.insert( plan.proxy_seq, - shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms), + shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms), ); } session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); diff --git a/backend/app/src/api/endpoints/hls_api/mod.rs b/backend/app/src/api/endpoints/hls_api/mod.rs index 10c21bf12..928f36d44 100644 --- a/backend/app/src/api/endpoints/hls_api/mod.rs +++ b/backend/app/src/api/endpoints/hls_api/mod.rs @@ -110,10 +110,12 @@ pub(super) struct HlsProxyResourcePathParams { mod catchup; mod manifest; +mod origin_account_runtime; +mod origin_refresh_request; mod owner_token; mod segment; mod session; - +use self::origin_account_runtime::prepare_hls_origin_runtime; pub(in crate::api) use catchup::*; pub(in crate::api) use manifest::*; pub(in crate::api) use owner_token::*; diff --git a/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs b/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs new file mode 100644 index 000000000..9691336c7 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/origin_account_runtime.rs @@ -0,0 +1,719 @@ +#![allow(clippy::wildcard_imports)] +use super::{ + origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, + session::*, + *, +}; + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(super) async fn prepare_hls_origin_runtime( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + { + let current = session.read().await; + if current.origin_source.input_id != input.id + || (!current.origin_source.input_name.is_empty() && current.origin_source.input_name != input.name) + || current.origin_account_binding.as_ref().is_some_and(|binding| binding.input_name != input.name) + { + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(current.origin_account_binding.as_ref()), + }); + } + } + advance_hls_origin_account_lifecycle(app_state, session, now_ms).await; + // Healthy preparation does not queue behind account replacement. Preserve the manifest + // singleflight boundary when many clients arrive while a qualified window awaits media. + if let Some(current_input) = + app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name) + { + let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let binding = observed.binding.clone(); + if let Some(binding) = binding.filter(|binding| { + binding.is_active() + && hls_origin_account_status(&app_state.hls_ctx(), binding) == HlsOriginAccountStatus::Known + }) { + let prepared = prepared_hls_origin_runtime_for_known_binding( + app_state, + ¤t_input, + raw_request_url, + session_entry_url, + &binding, + ) + .await; + let mut current = session.write().await; + if !current.is_gc_marked_for_removal() && observed.matches_session(¤t) { + if current.origin_account_runtime.is_none() { + current.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime { + app_state: Arc::downgrade(app_state), + input_id: input.id, + input_name: Arc::clone(&input.name), + raw_request_url: raw_request_url.to_string(), + entry_url: session_entry_url.to_string(), + fingerprint: fingerprint.clone(), + })); + } + return Ok(PreparedHlsOriginRuntime::published( + prepared, + HlsOriginAccountSnapshot::from_session(¤t), + )); + } + } + } + let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let _prepare_guard = prepare_lock.lock().await; + let Some(input) = app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name) + else { + let mut session = session.write().await; + if session.origin_account_rebind.is_allowed_now(now_ms) { + if let Some(binding) = session.origin_account_binding.clone() { + session.origin_account_rebind.mark_attempt_started(binding.account_name, now_ms); + session.origin_account_rebind.mark_failed(now_ms); + } + } + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(session.origin_account_binding.as_ref()), + }); + }; + session.write().await.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime { + app_state: Arc::downgrade(app_state), + input_id: input.id, + input_name: Arc::clone(&input.name), + raw_request_url: raw_request_url.to_string(), + entry_url: session_entry_url.to_string(), + fingerprint: fingerprint.clone(), + })); + let input = input.as_ref(); + advance_hls_origin_account_lifecycle(app_state, session, now_ms).await; + let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let previous = observed.binding.clone(); + let mut prepared = prepare_hls_origin_runtime_locked( + app_state, + session, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + work_kind, + work_class, + now_ms, + ) + .await?; + let snapshot = publish_hls_origin_account_with_status(app_state, session, |current| { + let snapshot = if let Some(snapshot) = prepared.prepared_snapshot.take() { + if current.is_gc_marked_for_removal() || !snapshot.matches_session(current) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + snapshot + } else { + if current.is_gc_marked_for_removal() || !observed.matches_session(current) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + if let Some(mut binding) = prepared.binding_to_publish.take() { + binding.generation = previous.as_ref().map_or(0, |previous| previous.generation.saturating_add(1)); + binding.provider_binding_tag = prepared + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .and_then(|handle| handle.binding_tag); + if previous.as_ref().is_some_and(|previous| previous.account_name != binding.account_name) { + current.handoff_origin_account(binding); + } else { + current.replace_origin_account_binding(Some(binding)); + } + } + HlsOriginAccountSnapshot::from_session(current) + }; + Ok(snapshot) + }) + .await?; + Ok(PreparedHlsOriginRuntime::published(prepared, snapshot)) +} + +// Lock order: viewer rows, then HLS state. Publication and projection are synchronous once +// both locks are acquired: cancellation cannot publish an account without publishing its UI rows. +pub(super) async fn publish_hls_origin_account_with_status( + app_state: &Arc, + session: &HlsSessionHandle, + publish: impl FnOnce( + &mut tuliprox_hls::api::HlsSession, + ) -> Result, +) -> Result { + let shared_stream_id = hls_cache_shared_stream_id(&session.read().await.proxy_session_id); + let mut projection = app_state.active_users.lock_shared_stream_provider_projection(shared_stream_id).await; + let mut current = session.write().await; + let snapshot = publish(&mut current)?; + if !snapshot.matches_session(¤t) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + if let Some(binding) = snapshot.binding.as_ref().filter(|binding| binding.is_active()) { + projection.update_provider(&binding.account_name); + } + Ok(snapshot) +} + +async fn advance_hls_origin_account_lifecycle(app_state: &Arc, session: &HlsSessionHandle, now_ms: u64) { + promote_elapsed_hls_account_overlaps(app_state, now_ms).await; + detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; + reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn prepare_hls_origin_runtime_locked( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + proxy_session_id: &ProxySessionId, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + work_kind: HlsOriginWorkKind, + work_class: HlsOriginWorkClass, + now_ms: u64, +) -> Result { + let playback_kind = hls_session_origin_playback_kind(session).await; + let existing_binding = session.read().await.origin_account_binding.clone(); + let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached); + if reacquire_detached_binding && !session.read().await.origin_account_rebind.is_allowed_now(now_ms) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref()); + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_started(session, work_kind).await; + } + if let Some(binding) = existing_binding { + if binding.is_active() + || !matches!(hls_origin_account_status(&app_state.hls_ctx(), &binding), HlsOriginAccountStatus::Known) + { + match hls_origin_account_status(&app_state.hls_ctx(), &binding) { + stale_status @ (HlsOriginAccountStatus::Missing + | HlsOriginAccountStatus::Expired + | HlsOriginAccountStatus::Blocked) => { + return rebind_hls_origin_account( + app_state, + session, + input, + raw_request_url, + session_entry_url, + &binding, + stale_status, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await; + } + HlsOriginAccountStatus::Known => { + return Ok(prepared_hls_origin_runtime_for_known_binding( + app_state, + input, + raw_request_url, + session_entry_url, + &binding, + ) + .await); + } + } + } + } + + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + false, + work_kind, + work_class, + playback_kind, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)), + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + + if work_class.allows_speculative_overlap() { + if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + + if let Ok(prepared) = prepare_hls_speculative_origin_runtime( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + playback_kind, + now_ms, + ) + .await + { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + } else { + debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value()); + } + + if work_class.allows_grace() { + match prepare_hls_origin_runtime_with_new_account( + app_state, + input, + raw_request_url, + session_entry_url, + proxy_session_id, + fingerprint, + connection_kind, + priority, + true, + work_kind, + work_class, + playback_kind, + now_ms, + ) + .await + { + Ok(prepared) => { + if reacquire_detached_binding { + if let Some(binding) = prepared.binding_to_publish.as_ref() { + log_hls_origin_binding_reacquired(session, binding).await; + } + } + return Ok(prepared); + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { + return Err(HlsOriginRuntimeAcquireError::Fatal(status)) + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} + } + } else { + debug!( + "HLS origin account grace skipped: work_class={} reason=background-origin-work", + work_class.as_log_value() + ); + } + + if reacquire_detached_binding { + log_hls_origin_binding_reacquire_failed(session, "no-account-available").await; + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason }) +} + +#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_lines)] +pub(super) async fn rebind_hls_origin_account( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + raw_request_url: &str, + session_entry_url: &str, + stale_binding: &HlsOriginAccountBinding, + stale_status: HlsOriginAccountStatus, + fingerprint: &Fingerprint, + connection_kind: crate::api::model::ConnectionKind, + priority: i8, + playback_kind: PlaybackKind, + now_ms: u64, +) -> Result { + { + let mut session_guard = session.write().await; + if !session_guard.origin_account_rebind.is_allowed_now(now_ms) { + debug!( + "HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms); + } + + let safe_proxy_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + debug!( + "HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}", + safe_proxy_session, + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); + app_state.active_provider.clear_identified_provider_reservation( + &stale_binding.session_owner, + &stale_binding.account_name, + stale_binding.provider_binding_tag, + ); + let expected_work_generation = { + let mut session_guard = session.write().await; + if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { + binding.account_name == stale_binding.account_name + && binding.session_owner == stale_binding.session_owner + && binding.provider_binding_tag == stale_binding.provider_binding_tag + }) { + binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); + debug!( + "HLS origin binding detached: proxy_session={} account={} reason={}", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason() + ); + } + session_guard.invalidate_queued_origin_work(); + session_guard.activity.origin_work_generation + }; + + let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + false, + priority, + connection_kind, + Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)), + ) else { + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; + return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: hls_no_account_reason_for_binding(Some(stale_binding)), + }); + }; + + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_config) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_config()) + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(fetch_url) = resolve_hls_origin_fetch_url( + &app_state.app_config, + input, + raw_request_url, + session_entry_url, + Some(&provider_config), + ) + .await + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let Some(new_account_name) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_name()) + else { + drop(managed_handle); + mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; + + let mut new_binding = HlsOriginAccountBinding::rebound( + Arc::clone(&input.name), + new_account_name, + stale_binding.session_owner.clone(), + stale_binding.generation.saturating_add(1), + now_ms, + ); + new_binding.provider_binding_tag = managed_handle.handle().and_then(|handle| handle.binding_tag); + let prepared_snapshot = publish_hls_origin_account_with_status(app_state, session, |session_guard| { + if !publish_hls_origin_account_rebind( + session_guard, + stale_binding, + expected_work_generation, + new_binding.clone(), + ) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } + Ok(HlsOriginAccountSnapshot::from_session(session_guard)) + }) + .await?; + debug!( + "HLS origin account rebound: old_account={} new_account={}", + sanitize_sensitive_info(stale_binding.account_name.as_ref()), + sanitize_sensitive_info(new_binding.account_name.as_ref()) + ); + + Ok(HlsOriginRuntimeAllocation { + url_failover_provider: hls_url_failover_provider_for_origin_context( + input, + raw_request_url, + session_entry_url, + &fetch_url, + ), + runtime_provider_config: Some(provider_config), + fetch_url, + binding_to_publish: None, + prepared_snapshot: Some(prepared_snapshot), + preacquired_origin_account_handle: Some(managed_handle), + }) +} + +// Resolution may suspend after detaching. Publish only into the same account and work generation. +fn publish_hls_origin_account_rebind( + session: &mut tuliprox_hls::api::HlsSession, + expected: &HlsOriginAccountBinding, + expected_work_generation: u64, + replacement: HlsOriginAccountBinding, +) -> bool { + if session.is_gc_marked_for_removal() + || session.activity.origin_work_generation != expected_work_generation + || !session.origin_account_binding.as_ref().is_some_and(|current| { + current.input_name == expected.input_name + && current.account_name == expected.account_name + && current.session_owner == expected.session_owner + && current.generation == expected.generation + && current.provider_binding_tag == expected.provider_binding_tag + }) + { + return false; + } + session.handoff_origin_account(replacement); + session.origin_account_rebind.mark_success(); + true +} + +pub(super) async fn mark_hls_origin_rebind_failed( + session: &HlsSessionHandle, + stale_binding: &HlsOriginAccountBinding, + now_ms: u64, + reason: &str, +) { + let mut session_guard = session.write().await; + session_guard.origin_account_rebind.mark_failed(now_ms); + debug!( + "HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000", + safe_proxy_session_id(&session_guard.proxy_session_id), + sanitize_sensitive_info(stale_binding.account_name.as_ref()) + ); +} + +use std::{future::Future, pin::Pin, sync::Weak}; +use tuliprox_hls::api::{ + trigger_origin_refresh_sync, HlsBoundAccountAcquireErrorKind, HlsManifestAcceptanceDirective, + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + LiveHlsOriginEntry, +}; + +struct AppHlsOriginAccountRuntime { + app_state: Weak, + input_id: u16, + input_name: Arc, + raw_request_url: String, + entry_url: String, + fingerprint: Fingerprint, +} + +impl std::fmt::Debug for AppHlsOriginAccountRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AppHlsOriginAccountRuntime").field("input_id", &self.input_id).finish_non_exhaustive() + } +} + +impl HlsOriginAccountRuntime for AppHlsOriginAccountRuntime { + #[allow(clippy::too_many_lines)] + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>> + { + Box::pin(async move { + let app_state = self.app_state.upgrade().ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?; + let input = app_state + .app_config + .get_input_by_id(self.input_id) + .filter(|input| input.name == self.input_name) + .ok_or(HlsBoundAccountAcquireErrorKind::Missing)?; + let (proxy_id, policy, unavailable) = { + let session = session.read().await; + ( + session.proxy_session_id.clone(), + session.effective_origin_acquire_policy_or_default(), + session.is_gc_marked_for_removal(), + ) + }; + if unavailable + || !app_state + .hls + .proxy + .segment_worker_pool() + .access_leases() + .write() + .await + .has_usable_access_lease_for_session(&proxy_id, current_time_millis()) + { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } + let prepared = prepare_hls_origin_runtime( + &app_state, + session, + &input, + &self.raw_request_url, + &self.entry_url, + &proxy_id, + &self.fingerprint, + policy.connection_kind, + policy.priority, + if work == HlsOriginAccountWork::Manifest { + HlsOriginWorkKind::Manifest + } else { + HlsOriginWorkKind::Resource + }, + class, + current_time_millis(), + ) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable)?; + // Drop releases unused allocations synchronously, including cancellation after prepare. + let managed = prepared.preacquired_origin_account_handle; + let entry = LiveHlsOriginEntry::parse_with_provider_configs( + &prepared.fetch_url, + prepared.url_failover_provider, + prepared.runtime_provider_config, + ) + .ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?; + let snapshot = prepared.snapshot; + let (pending, origin_headers) = { + let current = session.read().await; + if current.is_gc_marked_for_removal() || !snapshot.matches_session(¤t) { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + (current.origin_account_manifest_pending.is_some(), current.origin_request_headers.clone()) + }; + drop(managed); + if work == HlsOriginAccountWork::Media && pending { + let origin_io = HlsOriginIoContext { + ctx: app_state.hls_ctx(), + client_addr: self.fingerprint.addr, + allow_grace: class.allows_grace(), + priority: policy.priority, + connection_kind: policy.connection_kind, + reservation_ttl_secs: session.read().await.account_overlap_timing().reservation_ttl_secs(), + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), + }; + let manifest_commit_requirement = tuliprox_hls::api::hls_manifest_commit_requirement( + session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + false, + current_time_millis(), + ) + .await; + let request = hls_origin_refresh_request( + &app_state, + session, + HlsPreparedOriginAccount { entry: entry.clone(), snapshot: snapshot.clone() }, + HlsOriginRefreshOptions { + headers: origin_headers, + origin_io, + manifest_commit_requirement, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: app_state.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + trigger_origin_refresh_sync(request).await; + if session.read().await.origin_account_manifest_pending.is_some() { + return Err(HlsBoundAccountAcquireErrorKind::Unavailable); + } + } + Ok(HlsPreparedOriginAccount { entry, snapshot }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tuliprox_hls::api::{HlsSession, HlsSessionKey}; + + #[test] + fn rebind_publication_rejects_superseded_work_binding_and_gc() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "publication"), b"secret", 0); + let expected = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("expired"), &session.proxy_session_id, 0); + session.replace_origin_account_binding(Some(expected.clone())); + let replacement = HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("replacement"), + expected.session_owner.clone(), + 1, + 1, + ); + session.invalidate_queued_origin_work(); + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 0, replacement.clone())); + assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name, expected.account_name); + assert!(session.origin_account_manifest_pending.is_none()); + session.origin_account_binding.as_mut().unwrap().generation = 7; + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone())); + assert_eq!(session.origin_account_binding.as_ref().unwrap().generation, 7); + session.replace_origin_account_binding(Some(expected.clone())); + session.mark_for_gc_removal(); + assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone())); + session.clear_gc_removal_mark(); + assert!(publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement)); + assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "replacement"); + assert_eq!(session.activity.origin_work_generation, 2); + assert_eq!(session.origin_account_manifest_pending, Some(1)); + } +} diff --git a/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs b/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs new file mode 100644 index 000000000..211f4b7b2 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/origin_refresh_request.rs @@ -0,0 +1,58 @@ +use super::{AppState, HlsOriginIoContext, HlsSessionHandle}; +use axum::http::HeaderMap; +use std::sync::Arc; +use tuliprox_hls::api::{ + HlsAccessLeaseId, HlsManifestAcceptanceDirective, HlsManifestCommitRequirement, HlsPostRefreshRuntime, + HlsPreparedOriginAccount, OriginRefreshRequest, RetryPolicy, +}; + +pub(super) struct HlsOriginRefreshOptions { + pub headers: HeaderMap, + pub origin_io: HlsOriginIoContext, + pub manifest_commit_requirement: HlsManifestCommitRequirement, + pub acceptance_directive: HlsManifestAcceptanceDirective, + pub access_lease_id: Option, + pub rewrite_secret: Vec, + pub now_ms: u64, +} + +pub(super) fn hls_origin_refresh_request( + app_state: &Arc, + session: &HlsSessionHandle, + prepared: HlsPreparedOriginAccount, + options: HlsOriginRefreshOptions, +) -> OriginRefreshRequest { + let mut origin_io = options.origin_io; + let origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone(); + origin_io.prepared_account = Some(prepared.snapshot); + OriginRefreshRequest { + app_config: Arc::clone(&app_state.app_config), + session: Arc::clone(session), + origin_entry: prepared.entry, + headers: options.headers, + origin_provider_session_headers, + disabled_headers: app_state.get_disabled_headers(), + client: app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), + hls_proxy: Arc::clone(&app_state.hls.proxy), + segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), + segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), + origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), + strip: app_state.hls.proxy.strip().clone(), + retry_policy: RetryPolicy::default(), + reverse_proxy_rewrite_secret: options.rewrite_secret, + transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), + manifest_commit_requirement: options.manifest_commit_requirement, + fresh_manifest_requirement_generation: None, + refresh_identity: None, + acceptance_directive: options.acceptance_directive, + access_lease_id: options.access_lease_id, + now_ms: options.now_ms, + origin_io: Some(origin_io), + post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: app_state.hls_ctx().downgrade() }), + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/access.rs b/backend/app/src/api/endpoints/hls_api/segment/access.rs index 89f18c52b..f3766b16f 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/access.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/access.rs @@ -186,7 +186,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_cached_segment( match &lease.playback_mode { HlsLeasePlaybackMode::Live => true, HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => false, } } diff --git a/backend/app/src/api/endpoints/hls_api/segment/fallback.rs b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs index fca2112b1..dbc7af969 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/fallback.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs @@ -94,7 +94,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_runtime_or_standalone_custom return fallback_status.into_response(); }; match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { return response; } @@ -196,7 +198,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_origin_runtime_resource_failure_re err: HlsOriginRuntimeAcquireError, ) -> axum::response::Response { match err { - HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(), + HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => { + hls_canonical_status_response(StatusCode::SERVICE_UNAVAILABLE) + } HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status), } } diff --git a/backend/app/src/api/endpoints/hls_api/segment/registration.rs b/backend/app/src/api/endpoints/hls_api/segment/registration.rs index 505a04d2e..53d70abff 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/registration.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/registration.rs @@ -27,6 +27,10 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_hls_cache_stream_register access: &HlsAccessContext, session: &HlsSessionHandle, ) -> Option { + // Share the existing origin-publication gate: a registration cannot retain an old + // account snapshot across a handover and then overwrite its projected provider. + let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let _prepare_guard = prepare_lock.lock().await; let (proxy_session_id, origin_source, origin_account_binding) = { let session = session.read().await; if session.is_gc_marked_for_removal() { diff --git a/backend/app/src/api/endpoints/hls_api/segment/resources.rs b/backend/app/src/api/endpoints/hls_api/segment/resources.rs index f9c68d7ea..4b12d2c85 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/resources.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/resources.rs @@ -145,9 +145,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_resource( match &lease.playback_mode { HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await, HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await, - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - StatusCode::NOT_FOUND.into_response() - } + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => StatusCode::NOT_FOUND.into_response(), } } @@ -226,7 +226,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc }; let resource = cache_resolution.resource; let origin_headers = cache_resolution.origin_headers; - let origin_provider_session_headers = cache_resolution.origin_provider_session_headers; let cache_action = cache_resolution.action; match cache_action { @@ -266,7 +265,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc resource, cache_action, origin_headers, - origin_provider_session_headers, cache_duration_ms, }) .await @@ -277,21 +275,14 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientPassthroughContext<'a> pub(in crate::api::endpoints::hls_api) resource: TransientResourceRef, pub(in crate::api::endpoints::hls_api) cache_action: HlsTransientObjectCacheAction, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, } pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_resource( context: HlsTransientPassthroughContext<'_>, ) -> axum::response::Response { - let HlsTransientPassthroughContext { - endpoint, - resource, - cache_action, - origin_headers, - origin_provider_session_headers, - cache_duration_ms, - } = context; + let HlsTransientPassthroughContext { endpoint, resource, cache_action, origin_headers, cache_duration_ms } = + context; if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action { return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext { app_state: endpoint.app_state, @@ -304,7 +295,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r resource_file: endpoint.resource_file, fetch_token: *fetch_token, origin_headers, - origin_provider_session_headers, range_header: endpoint.range_header, cache_duration_ms, }) @@ -321,7 +311,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r resource: &resource, resource_file: &endpoint.resource_file, origin_headers, - origin_provider_session_headers, range_header: endpoint.range_header.clone(), policy: policy.clone(), }) diff --git a/backend/app/src/api/endpoints/hls_api/segment/segments.rs b/backend/app/src/api/endpoints/hls_api/segment/segments.rs index 59393842f..f80c72b60 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/segments.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/segments.rs @@ -229,6 +229,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_terminal_segment( } HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, }, @@ -318,6 +319,8 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context( fingerprint: &Fingerprint, preacquired_provider_handle: Option, ) -> SegmentFetchContext { + let preacquired_provider_handle = preacquired_provider_handle + .map(|handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle)); let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = { let session = session.read().await; ( @@ -336,9 +339,12 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context( reservation_ttl_secs, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), }; if let Some(provider_handle) = preacquired_provider_handle { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + origin_io = origin_io.with_managed_provider_handle(provider_handle); } SegmentFetchContext { session: Arc::clone(session), diff --git a/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs index 82ffb062e..e8730f553 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs @@ -35,7 +35,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointOriginFetchReq pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef, pub(in crate::api::endpoints::hls_api) resource_file: &'a TransientResourceFile, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) range_header: Option, pub(in crate::api::endpoints::hls_api) policy: SegmentFetchPolicy, } @@ -54,14 +53,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_ no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(), use_manual_redirects: request.app_state.should_use_manual_redirects(), }; - let log_identity = { + let (log_identity, generation, provider_headers) = { let session = request.session.read().await; - HlsLogIdentity::from_session(&session) + ( + HlsLogIdentity::from_session(&session), + session.activity.origin_work_generation, + session.origin_provider_session_headers.clone(), + ) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: request.resource.resolved_origin_uri.clone(), origin_headers: request.origin_headers, - origin_provider_session_headers: request.origin_provider_session_headers, + origin_provider_session_headers: provider_headers, range_header: request.range_header, resource_file: request.resource_file.clone(), resource_kind: request.resource.kind, @@ -77,12 +80,19 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_ request.fingerprint, request.headers, &runtime_prepare_error, + HlsTransientOriginBindingSnapshot { resource: request.resource.clone(), generation }, ); let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await; let runtime_prepare_error = *runtime_prepare_error.lock().await; HlsTransientOriginFetchResult { result, runtime_prepare_error } } +#[derive(Clone)] +pub(in crate::api::endpoints::hls_api) struct HlsTransientOriginBindingSnapshot { + resource: TransientResourceRef, + generation: u64, +} + /// Builds the shared per-attempt prepare closure for transient origin fetches. /// Runtime acquire failures are captured in `runtime_prepare_error` and mapped /// to a provider-unavailable fetch error so the retry loop can proceed uniformly. @@ -93,6 +103,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( fingerprint: &Fingerprint, headers: &HeaderMap, runtime_prepare_error: &Arc>>, + snapshot: HlsTransientOriginBindingSnapshot, ) -> impl FnMut( HlsResourceFetchAttempt, ) -> futures::future::BoxFuture< @@ -112,6 +123,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( let fingerprint = fingerprint.clone(); let headers = headers.clone(); let runtime_prepare_error = Arc::clone(&runtime_prepare_error); + let snapshot = snapshot.clone(); async move { match prepare_hls_transient_origin_io_for_authorized_resource_work( &app_state, @@ -123,7 +135,17 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( ) .await { - Ok(guard) => Ok(guard), + Ok(guard) => { + let session = session.read().await; + if session.activity.origin_work_generation != snapshot.generation + || !session.transient.origin_resource_is_fetchable(&snapshot.resource, current_time_millis()) + { + return Err(HlsOriginResourceFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::StoreRace, + )); + } + Ok(guard) + } Err(err) => { *runtime_prepare_error.lock().await = Some(err); Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable)) @@ -146,7 +168,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointCacheFetchCont pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile, pub(in crate::api::endpoints::hls_api) fetch_token: TransientObjectFetchToken, pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, - pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, pub(in crate::api::endpoints::hls_api) range_header: Option, pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, } @@ -298,14 +319,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(), use_manual_redirects: context.app_state.should_use_manual_redirects(), }; - let log_identity = { + let (log_identity, generation, provider_headers) = { let session = context.session.read().await; - HlsLogIdentity::from_session(&session) + ( + HlsLogIdentity::from_session(&session), + session.activity.origin_work_generation, + session.origin_provider_session_headers.clone(), + ) }; let fetch_request = HlsTransientOriginFetchRequest { resolved_origin_uri: context.resource.resolved_origin_uri.clone(), origin_headers: context.origin_headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), + origin_provider_session_headers: provider_headers, range_header: None, resource_file: context.resource_file.clone(), resource_kind: context.resource.kind, @@ -336,6 +361,7 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin context.fingerprint, context.headers, &runtime_prepare_error, + HlsTransientOriginBindingSnapshot { resource: context.resource.clone(), generation }, ); let final_failure = match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt) diff --git a/backend/app/src/api/endpoints/hls_api/session.rs b/backend/app/src/api/endpoints/hls_api/session.rs index c19a53c7a..3347980da 100644 --- a/backend/app/src/api/endpoints/hls_api/session.rs +++ b/backend/app/src/api/endpoints/hls_api/session.rs @@ -378,13 +378,37 @@ pub(super) fn hls_url_failover_provider_for_origin_context( } pub(super) struct PreparedHlsOriginRuntime { + pub(super) fetch_url: String, + pub(super) runtime_provider_config: Option>, + pub(super) url_failover_provider: Option>, + pub(super) snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot, + pub(super) preacquired_origin_account_handle: Option, +} + +impl PreparedHlsOriginRuntime { + pub(super) fn published( + allocation: HlsOriginRuntimeAllocation, + snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot, + ) -> Self { + Self { + fetch_url: allocation.fetch_url, + runtime_provider_config: allocation.runtime_provider_config, + url_failover_provider: allocation.url_failover_provider, + snapshot, + preacquired_origin_account_handle: allocation.preacquired_origin_account_handle, + } + } +} + +pub(super) struct HlsOriginRuntimeAllocation { pub(super) fetch_url: String, pub(super) runtime_provider_config: Option>, // URL failover comes from source.yml provider:// resolution. Origin-account // binding/handles are runtime account reservations and must stay separate. pub(super) url_failover_provider: Option>, - pub(super) origin_account_binding_to_store: Option, - pub(super) preacquired_origin_account_handle: Option, + pub(super) binding_to_publish: Option, + pub(super) prepared_snapshot: Option, + pub(super) preacquired_origin_account_handle: Option, } pub(super) fn effective_hls_url_failover_provider_for_fetch_url( @@ -626,194 +650,8 @@ pub(super) fn hls_soft_overlap_delay_ms( /// /// Archive-backed sessions must keep catchup semantics instead of plain live HLS; /// live sessions keep the live HLS kind. -async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind { - if session.read().await.key.archive_reference.is_some() { - PlaybackKind::Catchup - } else { - PlaybackKind::LiveHls - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(super) async fn prepare_hls_origin_runtime( - app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - raw_request_url: &str, - session_entry_url: &str, - proxy_session_id: &ProxySessionId, - fingerprint: &Fingerprint, - connection_kind: crate::api::model::ConnectionKind, - priority: i8, - work_kind: HlsOriginWorkKind, - work_class: HlsOriginWorkClass, - now_ms: u64, -) -> Result { - let playback_kind = hls_session_origin_playback_kind(session).await; - promote_elapsed_hls_account_overlaps(app_state, now_ms).await; - detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await; - reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await; - let existing_binding = session.read().await.origin_account_binding.clone(); - let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached); - let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref()); - if reacquire_detached_binding { - log_hls_origin_binding_reacquire_started(session, work_kind).await; - } - if let Some(binding) = existing_binding { - if binding.is_active() { - match hls_origin_account_status(&app_state.hls_ctx(), &binding) { - stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => { - return rebind_hls_origin_account( - app_state, - session, - input, - raw_request_url, - session_entry_url, - &binding, - stale_status, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await; - } - HlsOriginAccountStatus::Known => { - return Ok(prepared_hls_origin_runtime_for_known_binding( - app_state, - input, - raw_request_url, - session_entry_url, - &binding, - ) - .await); - } - } - } - } - - match prepare_hls_origin_runtime_with_new_account( - app_state, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - false, - work_kind, - work_class, - playback_kind, - now_ms, - ) - .await - { - Ok(prepared) => { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)), - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} - } - - if work_class.allows_speculative_overlap() { - if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime( - app_state, - session, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await - { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - - if let Ok(prepared) = prepare_hls_speculative_origin_runtime( - app_state, - session, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - playback_kind, - now_ms, - ) - .await - { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - } else { - debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value()); - } - - if work_class.allows_grace() { - match prepare_hls_origin_runtime_with_new_account( - app_state, - input, - raw_request_url, - session_entry_url, - proxy_session_id, - fingerprint, - connection_kind, - priority, - true, - work_kind, - work_class, - playback_kind, - now_ms, - ) - .await - { - Ok(prepared) => { - if reacquire_detached_binding { - if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() { - log_hls_origin_binding_reacquired(session, binding).await; - } - } - return Ok(prepared); - } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { - return Err(HlsOriginRuntimeAcquireError::Fatal(status)) - } - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {} - } - } else { - debug!( - "HLS origin account grace skipped: work_class={} reason=background-origin-work", - work_class.as_log_value() - ); - } - - if reacquire_detached_binding { - log_hls_origin_binding_reacquire_failed(session, "no-account-available").await; - } - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason }) +pub(super) async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind { + session.read().await.origin_playback_kind() } #[allow(clippy::too_many_arguments)] @@ -831,7 +669,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( work_class: HlsOriginWorkClass, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let session_owner = build_hls_origin_session_owner(proxy_session_id); let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( &input.name, @@ -852,8 +690,13 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( }); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -866,7 +709,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -876,7 +719,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; @@ -896,7 +739,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( grace_state ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -905,8 +748,9 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -916,7 +760,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( raw_request_url: &str, session_entry_url: &str, binding: &HlsOriginAccountBinding, -) -> PreparedHlsOriginRuntime { +) -> HlsOriginRuntimeAllocation { let runtime_provider_config = app_state.active_provider.find_provider_config(&binding.account_name); let fetch_url = resolve_hls_origin_fetch_url( &app_state.app_config, @@ -928,7 +772,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( .await .unwrap_or_else(|| session_entry_url.to_string()); - PreparedHlsOriginRuntime { + HlsOriginRuntimeAllocation { runtime_provider_config, url_failover_provider: hls_url_failover_provider_for_origin_context( input, @@ -937,7 +781,8 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding( &fetch_url, ), fetch_url, - origin_account_binding_to_store: None, + binding_to_publish: None, + prepared_snapshot: None, preacquired_origin_account_handle: None, } } @@ -995,14 +840,15 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A timing.target_duration_ms ); let starting_window_ms = tuliprox_session::STARTING_LEASE_TTL_SECS.saturating_mul(1000); - let media_window_expired = session_guard.activity.last_delivered_media_at_ms.map_or_else( - || now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms), - |last| { - now_ms - > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms) - }, - ); - if !media_window_expired + // A retained session can contain delivery from an earlier playback. + // Give each newly published binding its own bounded start window; + // healthy preparation leaves pinned_at_ms unchanged. + let starting_window_expired = now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms); + let media_window_expired = session_guard.activity.last_delivered_media_at_ms.is_none_or(|last| { + now_ms > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms) + }); + if !starting_window_expired + || !media_window_expired || session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { continue; @@ -1033,7 +879,6 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(super) async fn prepare_hls_origin_policy_preempt_runtime( app_state: &Arc, - new_session: &HlsSessionHandle, input: &ConfigInput, raw_request_url: &str, session_entry_url: &str, @@ -1043,7 +888,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( priority: i8, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms); let Some(candidate) = find_hls_origin_policy_preempt_candidate(app_state, input, proxy_session_id, request_policy, now_ms).await @@ -1087,8 +932,13 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( debug!("HLS origin policy preemption denied: reason=exact-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1102,7 +952,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1113,7 +963,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( &provider_handle.allocation, now_ms, ) else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=invalid-allocation"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1140,16 +990,12 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( } } if !detached_victim { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS origin policy preemption denied: reason=stale-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); } - { - let mut session_guard = new_session.write().await; - session_guard.replace_origin_account_binding(Some(binding.clone())); - } debug!( "HLS origin policy preempted: account={} victim_proxy_session={} winner_proxy_session={} victim_kind={:?} victim_priority={} request_kind={:?} request_priority={}", sanitize_sensitive_info(candidate.account_name.as_ref()), @@ -1167,7 +1013,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( HlsOriginAccountDetachedReason::PreemptedByHigherPriority.as_log_reason() ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -1176,8 +1022,9 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -1283,7 +1130,6 @@ pub(super) fn hls_origin_policy_preempt_candidate_is_better( #[allow(clippy::too_many_arguments)] pub(super) async fn prepare_hls_speculative_origin_runtime( app_state: &Arc, - new_session: &HlsSessionHandle, input: &ConfigInput, raw_request_url: &str, session_entry_url: &str, @@ -1293,7 +1139,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( priority: i8, playback_kind: PlaybackKind, now_ms: u64, -) -> Result { +) -> Result { let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else { debug!("HLS account overlap denied: reason=no-soft-active-candidate"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1316,8 +1162,13 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( debug!("HLS account overlap denied: reason=speculative-acquire-failed"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); }; + let managed_handle = + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle); + let Some(provider_handle) = managed_handle.handle() else { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + }; let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=missing-provider-config"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1331,7 +1182,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( ) .await else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + drop(managed_handle); restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await; debug!("HLS account overlap denied: reason=invalid-origin-url"); return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); @@ -1344,10 +1195,6 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( candidate.reclaim_until_ms, now_ms, ); - { - let mut session_guard = new_session.write().await; - session_guard.replace_origin_account_binding(Some(binding.clone())); - } debug!( "HLS account overlap granted: account={} victim_proxy_session={} winner_proxy_session={} reclaim_until_ms={} eligible_after_ms={} delay_ms={} tuliprox_target_user_connections={} origin_input_account_connections={}", sanitize_sensitive_info(candidate.account_name.as_ref()), @@ -1359,7 +1206,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( candidate.tuliprox_target_user_connection_capacity, candidate.origin_input_account_connection_capacity ); - Ok(PreparedHlsOriginRuntime { + Ok(HlsOriginRuntimeAllocation { url_failover_provider: hls_url_failover_provider_for_origin_context( input, raw_request_url, @@ -1368,8 +1215,9 @@ pub(super) async fn prepare_hls_speculative_origin_runtime( ), runtime_provider_config: Some(provider_config), fetch_url, - origin_account_binding_to_store: Some(binding), - preacquired_origin_account_handle: Some(provider_handle), + binding_to_publish: Some(binding), + prepared_snapshot: None, + preacquired_origin_account_handle: Some(managed_handle), }) } @@ -1650,154 +1498,6 @@ pub(super) async fn promote_elapsed_hls_account_overlaps(app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - raw_request_url: &str, - session_entry_url: &str, - stale_binding: &HlsOriginAccountBinding, - stale_status: HlsOriginAccountStatus, - fingerprint: &Fingerprint, - connection_kind: crate::api::model::ConnectionKind, - priority: i8, - playback_kind: PlaybackKind, - now_ms: u64, -) -> Result { - { - let mut session_guard = session.write().await; - if !session_guard.origin_account_rebind.is_allowed_now(now_ms) { - debug!( - "HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms); - } - - let safe_proxy_session = { - let session_guard = session.read().await; - safe_proxy_session_id(&session_guard.proxy_session_id) - }; - debug!( - "HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}", - safe_proxy_session, - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); - app_state.active_provider.clear_identified_provider_reservation( - &stale_binding.session_owner, - &stale_binding.account_name, - stale_binding.provider_binding_tag, - ); - { - let mut session_guard = session.write().await; - if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| { - binding.account_name == stale_binding.account_name - && binding.session_owner == stale_binding.session_owner - && binding.provider_binding_tag == stale_binding.provider_binding_tag - }) { - binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms); - debug!( - "HLS origin binding detached: proxy_session={} account={} reason={}", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()), - HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason() - ); - } - session_guard.invalidate_queued_origin_work(); - } - - let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session( - &input.name, - &fingerprint.addr, - false, - priority, - connection_kind, - Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)), - ) else { - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await; - return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { - reason: hls_no_account_reason_for_binding(Some(stale_binding)), - }); - }; - - let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let Some(fetch_url) = resolve_hls_origin_fetch_url( - &app_state.app_config, - input, - raw_request_url, - session_entry_url, - Some(&provider_config), - ) - .await - else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let Some(new_account_name) = provider_handle.allocation.get_provider_name() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await; - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - - let new_binding = HlsOriginAccountBinding::rebound( - Arc::clone(&input.name), - new_account_name, - stale_binding.session_owner.clone(), - stale_binding.generation.saturating_add(1), - now_ms, - ); - { - let mut session_guard = session.write().await; - session_guard.replace_origin_account_binding(Some(new_binding.clone())); - session_guard.origin_account_rebind.mark_success(); - } - debug!( - "HLS origin account rebound: old_account={} new_account={}", - sanitize_sensitive_info(stale_binding.account_name.as_ref()), - sanitize_sensitive_info(new_binding.account_name.as_ref()) - ); - - Ok(PreparedHlsOriginRuntime { - url_failover_provider: hls_url_failover_provider_for_origin_context( - input, - raw_request_url, - session_entry_url, - &fetch_url, - ), - runtime_provider_config: Some(provider_config), - fetch_url, - origin_account_binding_to_store: None, - preacquired_origin_account_handle: Some(provider_handle), - }) -} - -pub(super) async fn mark_hls_origin_rebind_failed( - session: &HlsSessionHandle, - stale_binding: &HlsOriginAccountBinding, - now_ms: u64, - reason: &str, -) { - let mut session_guard = session.write().await; - session_guard.origin_account_rebind.mark_failed(now_ms); - debug!( - "HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000", - safe_proxy_session_id(&session_guard.proxy_session_id), - sanitize_sensitive_info(stale_binding.account_name.as_ref()) - ); -} - #[allow(clippy::too_many_arguments)] pub(super) async fn prepare_hls_cache_user_session( app_state: &Arc, @@ -1995,6 +1695,7 @@ pub(super) async fn hls_segment_request_requires_origin_work( && entry.origin_fetch_ref.is_some() } +#[cfg(test)] pub(super) async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool { let session = session.read().await; session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) @@ -2008,7 +1709,7 @@ pub(super) fn hls_transient_origin_binding_requires_runtime_prepare( || (binding.is_active() && matches!( hls_origin_account_status(hls_ctx, binding), - HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired + HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired | HlsOriginAccountStatus::Blocked )) } @@ -2021,11 +1722,22 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work( work_kind: HlsOriginWorkKind, now_ms: u64, ) -> Result, HlsOriginRuntimeAcquireError> { - if !hls_origin_binding_needs_reacquire(session).await { + let runtime = session.read().await.origin_account_runtime.clone(); + if let Some(runtime) = runtime { + runtime + .prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand) + .await + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; return Ok(None); } - if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + if !session + .read() + .await + .origin_account_binding + .as_ref() + .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&app_state.hls_ctx(), binding)) + { + return Ok(None); } let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) .await @@ -2047,13 +1759,17 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work( now_ms, ) .await?; - if let Some(binding) = prepared_origin.origin_account_binding_to_store { - session.write().await.replace_origin_account_binding(Some(binding)); + drop(prepared_origin); + let runtime = session.read().await.origin_account_runtime.clone(); + if let Some(runtime) = runtime { + runtime + .prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand) + .await + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; } - Ok(prepared_origin.preacquired_origin_account_handle) + Ok(None) } -#[allow(clippy::too_many_lines)] pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work( app_state: &Arc, session: &HlsSessionHandle, @@ -2062,120 +1778,55 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work req_headers: &HeaderMap, now_ms: u64, ) -> Result, HlsOriginRuntimeAcquireError> { - let hls_ctx = app_state.hls_ctx(); - let existing_binding = session.read().await.origin_account_binding.clone(); - let origin_policy = hls_effective_origin_acquire_policy(session).await; - let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await; - if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) { - match hls_origin_account_status(&hls_ctx, binding) { - HlsOriginAccountStatus::Known => { - let origin_io = HlsOriginIoContext { - ctx: hls_ctx.clone(), - client_addr: fingerprint.addr, - allow_grace: HlsOriginWorkClass::Demand.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, - preacquired_provider_handle: None, - started_generation: None, - }; - if let Ok(lease_guard) = begin_hls_origin_account_io_bounded( - &origin_io, - session, - binding, - hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms), - ) - .await - { - let (started_generation, active_origin_work_count) = { - let mut session = session.write().await; - let count = Arc::clone(&session.activity.active_origin_work_count); - (session.start_origin_work(), count) - }; - return Ok(Some(HlsTransientOriginIoGuard::new( - Arc::clone(session), - active_origin_work_count, - origin_io, - lease_guard, - started_generation, - ))); - } - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {} - } - } - - if !existing_binding - .as_ref() - .is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, binding)) - { - return Ok(None); - } - if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - } - let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers) - .await - .map_err(HlsOriginRuntimeAcquireError::Fatal)?; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let prepared_origin = prepare_hls_origin_runtime( + prepare_hls_origin_binding_for_authorized_resource_work( app_state, session, - &request_context.input, - &request_context.hls_url, - request_context.session_entry_url.as_str(), - &proxy_session_id, + access_context, fingerprint, - origin_policy.connection_kind, - origin_policy.priority, + req_headers, HlsOriginWorkKind::Resource, - HlsOriginWorkClass::Demand, now_ms, ) .await?; - if let Some(binding) = prepared_origin.origin_account_binding_to_store { - session.write().await.replace_origin_account_binding(Some(binding)); - } - let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle else { + let Some(binding) = session.read().await.origin_account_binding.clone() else { return Ok(None); }; - let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active) - else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); + if binding.is_detached() { return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; + } + let policy = hls_effective_origin_acquire_policy(session).await; let origin_io = HlsOriginIoContext { - ctx: hls_ctx, + ctx: app_state.hls_ctx(), client_addr: fingerprint.addr, allow_grace: HlsOriginWorkClass::Demand.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, + priority: policy.priority, + connection_kind: policy.connection_kind, + reservation_ttl_secs: hls_origin_account_reservation_ttl_secs_for_session(session).await, preacquired_provider_handle: None, started_generation: None, - } - .with_preacquired_provider_handle(provider_handle); - let Ok(lease_guard) = begin_hls_origin_account_io_bounded( + prepared_account: None, + playback_kind: session.read().await.origin_playback_kind(), + account_runtime: session.read().await.origin_account_runtime.clone(), + }; + let lease_guard = begin_hls_origin_account_io_bounded( &origin_io, session, &binding, hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms), ) .await - else { - return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); - }; - let (started_generation, active_origin_work_count) = { + .map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?; + let (generation, active_count) = { let mut session = session.write().await; + if !session.origin_account_binding.as_ref().is_some_and(|current| { + current.generation == binding.generation + && current.account_name == binding.account_name + && current.is_active() + }) { + return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE)); + } let count = Arc::clone(&session.activity.active_origin_work_count); (session.start_origin_work(), count) }; - Ok(Some(HlsTransientOriginIoGuard::new( - Arc::clone(session), - active_origin_work_count, - origin_io, - lease_guard, - started_generation, - ))) + Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), active_count, origin_io, lease_guard, generation))) } diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index 3071701ba..066c5194d 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -78,15 +78,20 @@ use tuliprox_hls::api::{ HlsCachedManifestOptions, HlsLeaseManifestSnapshotInput, LiveHlsOriginEntry, }; +mod origin_account_rebind; mod owner_token; +mod reduced_retry; +mod same_account_resume; mod archive; mod bandwidth; mod encrypted_media; mod entry_identity; mod headers; +mod late_custom_response; mod lease_lifecycle; mod manifest_entry_freshness; +mod manifest_short_window; mod manifest_startup; mod observability; mod owner_handoff; diff --git a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs index 08b417e9c..212ff5264 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs @@ -163,7 +163,7 @@ async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_sessi .is_some()); assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none()); assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -398,7 +398,7 @@ async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_h assert!(!response.headers().contains_key(header::LOCATION)); assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl"); assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -424,7 +424,7 @@ async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_targ assert_eq!(response.status(), StatusCode::BAD_REQUEST); assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] @@ -545,7 +545,7 @@ async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_c assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); assert!(reservation.selected_provider_config.is_some()); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); app_state.connection_manager.release_provider_handle(reservation.provider_handle); // A manifest-only entry reservation is still unconfirmed, so it must not block a @@ -1053,7 +1053,7 @@ async fn hls_entry_for_input_in_maintenance_opens_no_session() { // No maintenance clip is configured here, so the configured fallback status answers. assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input"); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs new file mode 100644 index 000000000..552348c2e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs @@ -0,0 +1,259 @@ +use super::*; + +#[tokio::test] +async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() { + let fixture = terminal_tail::publication_late_fixture().await; + let now_ms = super::super::current_time_millis(); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("published live lease"); + let snapshot = lease.last_manifest_snapshot.expect("published snapshot"); + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let requests_before = fixture.origin.manifest_request_count(); + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); + assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1))); + assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000))); + assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + assert!(!body.contains("/hls/shared/live/")); + let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::>(); + assert_eq!(media_uris.len(), 12); + let (first, second) = tokio::join!( + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), + ); + for response in [first, second] { + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, body.as_bytes()); + } + let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; + assert_eq!(media.status(), StatusCode::OK); + assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t"); + let bytes = response_body(media).await; + assert!(!bytes.is_empty()); + let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await; + assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(range).await, bytes.slice(..188)); + let router = + crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state)); + let head = router + .oneshot( + Request::builder() + .method(Method::HEAD) + .uri(media_uris[0]) + .extension(ConnectInfo(test_addr())) + .body(Body::empty()) + .expect("HEAD request"), + ) + .await + .expect("HEAD response"); + assert_eq!(head.status(), StatusCode::OK); + assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string()); + assert!(response_body(head).await.is_empty()); + fixture.app_state.app_config.custom_stream_response.store(None); + let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; + assert_eq!(replay.status(), StatusCode::OK); + assert_eq!(response_body(replay).await, bytes); + let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease"); + assert_eq!( + custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(), + StatusCode::NOT_FOUND + ); + let wrong_index = media_uris[0].replace("/0.ts", "/12.ts"); + assert_eq!( + custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(), + StatusCode::NOT_FOUND + ); + let late = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal lease"); + assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert_eq!(late.playback_cursor, lease.playback_cursor); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert_eq!(fixture.origin.manifest_request_count(), requests_before); +} + +#[tokio::test] +async fn late_custom_response_without_configured_asset_retains_error_response() { + let fixture = terminal_tail::publication_late_fixture().await; + fixture.app_state.app_config.custom_stream_response.store(None); + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response_body(response).await.is_empty()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal unavailable lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); +} + +#[tokio::test] +async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() { + let (fixture, media_uri) = committed_late_http_fixture().await; + for (method, range, status) in [ + (Method::GET, None, StatusCode::OK), + (Method::HEAD, None, StatusCode::OK), + (Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT), + ] { + let now_ms = super::super::current_time_millis(); + let previous_access_ms = now_ms.saturating_sub(50_000); + assert!(fixture + .app_state + .hls + .proxy + .activate_access_lease( + &fixture.access_lease_id, + &fixture.proxy_session_id, + previous_access_ms, + HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let before = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("valid late lease"); + let response = + custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await; + assert_eq!(response.status(), status); + let after = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("renewed late lease"); + assert!(after.last_seen_at_ms >= now_ms); + assert!(after.valid_until_ms > before.valid_until_ms); + assert!(after.active_until_ms > before.active_until_ms); + assert_eq!(after.playback_mode, before.playback_mode); + assert_eq!(after.playback_cursor, before.playback_cursor); + assert_eq!(after.admission_generation, before.admission_generation); + } +} + +#[tokio::test] +async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() { + let (fixture, media_uri) = committed_late_http_fixture().await; + let now_ms = super::super::current_time_millis(); + assert!(fixture + .app_state + .hls + .proxy + .activate_access_lease( + &fixture.access_lease_id, + &fixture.proxy_session_id, + now_ms - 50_000, + HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let before = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("valid late lease"); + let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint"); + for (uri, range, expected) in [ + (media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND), + (media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND), + (media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND), + (media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE), + ] { + let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await; + assert_eq!(response.status(), expected); + } + fixture.app_state.app_config.api_proxy.store(None); + let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await; + assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status()); + let after = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("unchanged late lease"); + assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms); + assert_eq!(after.valid_until_ms, before.valid_until_ms); + assert_eq!(after.active_until_ms, before.active_until_ms); +} + +async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) { + let fixture = terminal_tail::publication_late_fixture().await; + { + let mut session = fixture.session.write().await; + session.segments.clear(); + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + } + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); + let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned(); + (fixture, media_uri) +} + +async fn custom_media_response(app_state: Arc, uri: &str, range: Option<&str>) -> Response { + custom_media_response_with_method(app_state, uri, range, Method::GET).await +} + +async fn custom_media_response_with_method( + app_state: Arc, + uri: &str, + range: Option<&str>, + method: Method, +) -> Response { + let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state); + let mut request = Request::builder().method(method).uri(uri); + if let Some(range) = range { + request = request.header(header::RANGE, range); + } + router + .oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request")) + .await + .expect("custom media response") +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs index ae6bf363c..db7809f38 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_entry_freshness.rs @@ -17,11 +17,14 @@ use std::sync::{ const ORIGIN_WINDOW_SEGMENTS: u64 = 6; const SEGMENT_BODY: &[u8] = &[0x47; 8_192]; -fn origin_playlist(head: u64) -> Vec { +fn origin_playlist(head: u64) -> Vec { origin_playlist_with_duration(head, 2) } + +pub(super) fn origin_playlist_with_duration(head: u64, target_duration: u64) -> Vec { use std::fmt::Write; - let mut body = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + let mut body = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{target_duration}\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); for sequence in head..head + ORIGIN_WINDOW_SEGMENTS { - let _ = write!(body, "#EXTINF:2.0,\nseg-{sequence}.ts\n"); + let _ = write!(body, "#EXTINF:{target_duration}.0,\nseg-{sequence}.ts\n"); } body.into_bytes() } @@ -61,6 +64,20 @@ async fn first_origin_sequence_for_new_lease( request_url: &str, lease: &str, ) -> u64 { + let body = open_manifest_for_new_lease(app_state, input, request_url, lease).await; + let session_key = super::super::build_hls_origin_source(input, "12345").session_key(); + let first_proxy_seq = manifest_media_sequence(&body); + let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session"); + let session = session.read().await; + session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence +} + +pub(super) async fn open_manifest_for_new_lease( + app_state: &Arc, + input: &ConfigInput, + request_url: &str, + lease: &str, +) -> String { let origin_source = super::super::build_hls_origin_source(input, "12345"); let session_key = origin_source.session_key(); let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); @@ -88,11 +105,7 @@ async fn first_origin_sequence_for_new_lease( .await .expect("hls cache should handle the live entrypoint"); assert_eq!(response.status(), StatusCode::OK, "lease {lease}"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - let first_proxy_seq = manifest_media_sequence(&body); - let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session"); - let session = session.read().await; - session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence + String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8") } /// Moves the retained session's last origin confirmation `idle_ms` into the diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs new file mode 100644 index 000000000..b2192e10f --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs @@ -0,0 +1,526 @@ +use super::{support::TestEncodedManifestOrigin, *}; +use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode}; +use std::sync::atomic::AtomicU64; +use tuliprox_hls::api::HlsFreshManifestRequiredReason; + +struct ShortWindowFixture { + _temp_dir: tempfile::TempDir, + origin: TestSegmentOrigin, + app_state: Arc, + input: ConfigInput, + request_url: String, + head: Arc, + count: Arc, + proxy_session_id: ProxySessionId, +} + +enum ShortWindowResources { + Plain, + EncryptedWithMap, + FirstReady { media_base_url: String }, +} + +async fn short_window_fixture(count: usize) -> ShortWindowFixture { + short_window_resource_fixture(count, ShortWindowResources::Plain).await +} + +async fn short_window_resource_fixture(count: usize, resources: ShortWindowResources) -> ShortWindowFixture { + let startup_mode = match &resources { + ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => HlsStartupMode::Conservative, + ShortWindowResources::FirstReady { .. } => HlsStartupMode::FirstReady, + }; + let head = Arc::new(AtomicU64::new(100)); + let count = Arc::new(AtomicUsize::new(count)); + let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count)); + let origin = spawn_test_binary_origin(Arc::new(move |path| { + let body = if path_has_extension(path, "m3u8") { + let head = origin_head.load(Ordering::SeqCst); + let mut body = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + let extension = match &resources { + ShortWindowResources::Plain | ShortWindowResources::FirstReady { .. } => "ts", + ShortWindowResources::EncryptedWithMap => { + body.push_str("#EXT-X-MAP:URI=\"init.mp4\"\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x01010101010101010101010101010101\n"); + "m4s" + } + }; + for offset in 0..origin_count.load(Ordering::SeqCst) { + let segment_url = match &resources { + ShortWindowResources::FirstReady { media_base_url } => { + format!("{media_base_url}/segment-{}.ts", head + u64::try_from(offset).unwrap()) + } + ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => { + format!("segment-{}.{extension}", head + u64::try_from(offset).unwrap()) + } + }; + writeln!(body, "#EXTINF:2.5,\n{segment_url}").unwrap(); + } + Arc::from(body.into_bytes()) + } else if path_has_extension(path, "mp4") { + Arc::from(&b"short-window-map"[..]) + } else if path_has_extension(path, "bin") { + Arc::from(AES_TEST_KEY_BYTES) + } else { + match &resources { + ShortWindowResources::Plain => Arc::from(&b"short-window-media"[..]), + ShortWindowResources::FirstReady { .. } => unreachable!("media uses the gated segment origin"), + ShortWindowResources::EncryptedWithMap => Arc::from(encrypt_test_aes128_cbc_pkcs7( + b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])), + } + }; + TestBinaryOriginResponse::new(StatusCode::OK, body) + })) + .await; + let temp_dir = tempfile::tempdir().unwrap(); + let input = ConfigInput { + id: 1, + name: Arc::from("short-window-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + let mut cache = HlsCacheConfigDto { + cache_path: Some(temp_dir.path().to_string_lossy().to_string()), + startup: HlsStartupConfigDto { mode: startup_mode, ..HlsStartupConfigDto::default() }, + ..HlsCacheConfigDto::default() + }; + cache.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly; + let config = Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(cache), + ..Default::default() + })), + ..Config::default() + }; + app_state.app_config.config.store(Arc::new(config)); + app_state.hls.proxy.update_config(&app_state.app_config).await; + configure_default_test_server(&app_state); + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "short-channel", &request_url)).await; + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let proxy_session_id = crate::api::model::build_proxy_session_id(&key, &app_state.get_encrypt_secret()); + ShortWindowFixture { _temp_dir: temp_dir, origin, app_state, input, request_url, head, count, proxy_session_id } +} + +/// Keeps the first media fetch pending until all lease entries have been checked. +async fn gated_segment_origin(gate: Arc) -> TestEncodedManifestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let origin_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let requests = Arc::clone(&origin_requests); + let gate = Arc::clone(&gate); + tokio::spawn(async move { + let mut buffer = [0_u8; 2048]; + let read = socket.read(&mut buffer).await.unwrap(); + requests.lock().await.push(String::from_utf8_lossy(&buffer[..read]).to_string()); + let _permit = gate.acquire().await; + let body = [0x47; 8192]; + let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(headers.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + } + }); + TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task } +} + +async fn wait_for_shared_first_ready_refresh( + fixture: &ShortWindowFixture, + session: &HlsSessionHandle, + previous_requests: usize, +) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if fixture.origin.manifest_request_count() > previous_requests + && !session.read().await.origin_refresh.in_flight + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("additional leases complete one shared freshness request"); + assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1); +} + +async fn assert_first_ready_lease_media( + fixture: &ShortWindowFixture, + leases: &mut tokio::task::JoinSet, + minimum_media_sequence: u64, +) { + tokio::time::timeout(Duration::from_secs(5), async { + while let Some(result) = leases.join_next().await { + let body = result.unwrap(); + assert!(manifest_media_sequence(&body) >= minimum_media_sequence); + assert_eq!(media_uri_count(&body), 1); + for uri in body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x47; 8192]); + } + } + }) + .await + .expect("media publication releases every waiting lease"); +} + +#[tokio::test] +async fn first_ready_leases_share_qualification_while_first_segment_is_loading() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let media = gated_segment_origin(Arc::clone(&gate)).await; + let fixture = Arc::new( + short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() }) + .await, + ); + let mut leases = tokio::task::JoinSet::new(); + let first = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&first, "first-ready-0").await }); + let session = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if !media.requests.lock().await.is_empty() { + if let Some(session) = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await + { + if !session.read().await.origin_refresh.in_flight { + break session; + } + } + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("qualified origin has started its first segment fetch"); + let generation = { + let session = session.read().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert!(session.last_rendered_manifest.is_none()); + assert_eq!(session.origin_control.acceptance_generation.0, 1); + session.origin_control.acceptance_generation + }; + assert_eq!(fixture.origin.manifest_request_count(), 3); + for lease in 1..20 { + let fixture = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await }); + } + wait_for_shared_first_ready_refresh(&fixture, &session, 3).await; + assert!(session.read().await.last_rendered_manifest.is_none()); + assert_eq!(session.read().await.origin_control.acceptance_generation, generation); + gate.close(); + assert_first_ready_lease_media(&fixture, &mut leases, 0).await; + assert_eq!(session.read().await.origin_control.acceptance_generation, generation); +} + +#[tokio::test] +async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loading() { + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + let media = gated_segment_origin(Arc::clone(&gate)).await; + let fixture = Arc::new( + short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() }) + .await, + ); + let previous_body = open_short_lease(&fixture, "revalidation-baseline").await; + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + let previous_commit = { + let mut session = session.write().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 1); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + session.activity.last_authorized_media_at_ms = None; + session.startup.as_mut().unwrap().revisions.clear(); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Expired; + } + session.last_normal_manifest_commit_identity().unwrap() + }; + assert_eq!(fixture.origin.manifest_request_count(), 3); + assert_eq!(gate.forget_permits(1), 1); + assert_eq!( + hls_manifest_commit_requirement( + &session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + true, + super::super::current_time_millis(), + ) + .await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + } + ); + fixture.head.store(101, Ordering::SeqCst); + let mut leases = tokio::task::JoinSet::new(); + let first = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&first, "revalidation-waiting-0").await }); + let revalidation = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if media.requests.lock().await.len() >= 2 { + let state = session.read().await; + if !state.origin_refresh.in_flight && state.origin_seq_highwater == Some(101) { + break; + } + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await; + let media_requests = media.requests.lock().await.len(); + { + let session = session.read().await; + assert!( + revalidation.is_ok(), + "revalidation requests={} generation={} highwater={:?} media_requests={media_requests} in_flight={} commit={:?}", + fixture.origin.manifest_request_count(), + session.origin_control.acceptance_generation.0, + session.origin_seq_highwater, + session.origin_refresh.in_flight, + session.last_normal_manifest_commit_identity(), + ); + } + assert_eq!(fixture.origin.manifest_request_count(), 5); + { + let session = session.read().await; + assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + } + assert_eq!( + hls_manifest_commit_requirement( + &session, + tuliprox_hls::api::HlsSessionStoreOutcome::Reused, + None, + true, + super::super::current_time_millis(), + ) + .await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + } + ); + for lease in 1..20 { + let fixture = Arc::clone(&fixture); + leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await }); + } + wait_for_shared_first_ready_refresh(&fixture, &session, 5).await; + assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2); + assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit)); + gate.close(); + assert_first_ready_lease_media(&fixture, &mut leases, manifest_media_sequence(&previous_body) + 1).await; + assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2); + assert!(session.read().await.last_normal_manifest_commit_identity().unwrap() > previous_commit); +} + +async fn open_short_lease(fixture: &ShortWindowFixture, lease: &str) -> String { + let access_lease_id = HlsAccessLeaseId(lease.to_string()); + let access = test_hls_access_context(fixture.proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &access_lease_id).await; + let response = super::super::try_hls_cache_canonical_manifest_response( + &fixture.app_state, + &test_fingerprint(), + &access, + &fixture.proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &fixture.request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&fixture.request_url), + input: &fixture.input, + origin_source: super::super::build_hls_origin_source(&fixture.input, "12345"), + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("shared HLS response"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(!body.contains("#EXT-X-ENDLIST"), "short origin must remain live: {body}"); + assert!(body.contains("/hls/shared/live/")); + body +} + +async fn assert_short_media(fixture: &ShortWindowFixture, body: &str) { + for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), b"short-window-media"); + } +} + +#[tokio::test] +async fn one_segment_origin_publishes_live_manifest_and_segment() { + let fixture = short_window_fixture(1).await; + let body = open_short_lease(&fixture, "short-one").await; + assert_eq!(media_uri_count(&body), 1); + assert_short_media(&fixture, &body).await; + assert_eq!(fixture.origin.manifest_request_count(), 3); +} + +#[tokio::test] +async fn two_segment_origin_publishes_live_manifest_and_segments() { + let fixture = short_window_fixture(2).await; + let body = open_short_lease(&fixture, "short-two").await; + assert_eq!(media_uri_count(&body), 2); + assert_short_media(&fixture, &body).await; + assert_eq!(fixture.origin.manifest_request_count(), 3); +} + +#[tokio::test] +async fn twenty_actual_leases_share_the_same_short_origin_qualification() { + let mut counts = Vec::new(); + for leases in [1, 20] { + let fixture = Arc::new(short_window_fixture(1).await); + let mut tasks = tokio::task::JoinSet::new(); + for lease in 0..leases { + let fixture = Arc::clone(&fixture); + tasks.spawn(async move { open_short_lease(&fixture, &format!("short-shared-{lease}")).await }); + } + while let Some(result) = tasks.join_next().await { + let body = result.unwrap(); + assert_eq!(media_uri_count(&body), 1); + assert_short_media(&fixture, &body).await; + } + counts.push(fixture.origin.manifest_request_count()); + let before_new_lease = fixture.origin.manifest_request_count(); + open_short_lease(&fixture, "later-short-lease").await; + assert_eq!(fixture.origin.manifest_request_count(), before_new_lease); + } + assert_eq!(counts, vec![3, 3]); +} + +#[tokio::test] +async fn ten_short_refresh_cycles_keep_serving_live_media_and_immutable_old_routes() { + assert_ten_short_refresh_cycles([3, 1, 3, 1, 3, 1, 3, 1, 3, 1]).await; +} + +#[tokio::test] +async fn continuously_advancing_one_segment_origin_serves_ten_refresh_cycles() { + assert_ten_short_refresh_cycles([1; 10]).await; +} + +async fn assert_ten_short_refresh_cycles(counts: [usize; 10]) { + let fixture = short_window_fixture(1).await; + let initial = open_short_lease(&fixture, "short-reloads").await; + assert_short_media(&fixture, &initial).await; + let uri = format!("/hls/shared/live/{}/short-reloads/manifest.m3u8", fixture.proxy_session_id.0); + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + let mut previous = initial.clone(); + let mut previous_last = session.read().await.last_rendered_manifest.as_ref().unwrap().last_proxy_seq; + for cycle in 1..=10 { + fixture.head.store(100 + cycle * 4, Ordering::SeqCst); + fixture.count.store(counts[usize::try_from(cycle - 1).unwrap()], Ordering::SeqCst); + { + let mut session = session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = 0; + } + let mut current = None; + for _ in 0..100 { + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(!body.contains("#EXT-X-ENDLIST")); + if body != previous { + current = Some(body); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let body = current.expect("normal refresh advances the short window"); + assert!(manifest_media_sequence(&body) >= manifest_media_sequence(&previous)); + previous = body.clone(); + assert_short_media(&fixture, &body).await; + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId("short-reloads".to_string()), + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .unwrap(); + let snapshot = lease.last_manifest_snapshot.unwrap(); + assert!(snapshot.last_proxy_seq > previous_last); + previous_last = snapshot.last_proxy_seq; + assert_eq!(snapshot.playlist_duration_ms, u64::try_from(media_uri_count(&body)).unwrap() * 2_500); + } + assert_eq!(fixture.origin.manifest_request_count(), 13); + assert_short_media(&fixture, &initial).await; + let invalid = uri.replace("short-reloads", "missing-lease"); + assert_ne!(get_response(Arc::clone(&fixture.app_state), &invalid, None).await.status(), StatusCode::OK); +} + +#[tokio::test] +async fn short_manifests_keep_map_key_and_segment_routes_on_later_publications() { + let fixture = short_window_resource_fixture(2, ShortWindowResources::EncryptedWithMap).await; + let initial = open_short_lease(&fixture, "short-resources").await; + let session = + fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap(); + for uri in initial.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + response_body(response).await; + } + let mut bodies = vec![initial]; + fixture.head.store(102, Ordering::SeqCst); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + let uri = format!("/hls/shared/live/{}/short-resources/manifest.m3u8", fixture.proxy_session_id.0); + for _ in 0..100 { + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + if body != bodies[0] { + bodies.push(body); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(bodies.len(), 2, "a second short publication must advance"); + for body in bodies { + assert!(!body.contains("#EXT-X-ENDLIST")); + for (tag, expected) in [("#EXT-X-MAP:", &b"short-window-map"[..]), ("#EXT-X-KEY:", AES_TEST_KEY_BYTES)] { + let uri = body + .lines() + .find(|line| line.starts_with(tag)) + .unwrap() + .split_once("URI=\"") + .unwrap() + .1 + .split_once('"') + .unwrap() + .0; + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), expected); + } + for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let response = get_response(Arc::clone(&fixture.app_state), uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_body(response).await.as_ref(), + encrypt_test_aes128_cbc_pkcs7(b"short-window-media", AES_TEST_KEY_BYTES, [1; 16]) + ); + } + } + assert_eq!(fixture.origin.manifest_request_count(), 4); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs index fd28c77e8..4b5153080 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs @@ -420,7 +420,7 @@ async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest() assert_eq!(binding.input_name.as_ref(), "test-input"); assert_eq!(binding.account_name.as_ref(), "test-input"); } - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs b/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs new file mode 100644 index 000000000..7e03aba87 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/origin_account_rebind.rs @@ -0,0 +1,1725 @@ +use super::{ + super::{prepare_hls_origin_runtime, HlsOriginWorkClass, HlsOriginWorkKind}, + *, +}; +use tuliprox_core::{model::ProviderAccountObservation, utils::current_time_millis}; +use tuliprox_hls::api::{ + hls_origin_account_status, HlsOriginAccountStatus, HlsOriginAccountWork, HlsSegmentFile, OriginMapKey, +}; + +#[derive(Clone, Copy, Debug)] +enum InvalidAccount { + ExpiredRoot, + ExpiredAlias, + RemovedAlias, + DisabledRoot, + DisabledAlias, + RuntimeRoot, + RuntimeAlias, +} + +fn reload_inputs(app: &Arc, inputs: Vec>) { + app.app_config.sources.store(Arc::new(SourcesConfig { + group_lookup: build_group_lookup(&inputs), + inputs, + ..app.app_config.sources.load_full().as_ref().clone() + })); + app.active_provider.update_config(&app.app_config); +} + +fn invalidate_account( + app: &Arc, + input: &mut ConfigInput, + invalid: InvalidAccount, +) -> (&'static str, HlsOriginAccountStatus) { + match invalid { + InvalidAccount::ExpiredRoot => input.exp_date = Some(1), + InvalidAccount::ExpiredAlias => input.aliases.as_mut().unwrap()[0].exp_date = Some(1), + InvalidAccount::RemovedAlias => input.aliases = None, + InvalidAccount::DisabledRoot => input.account_disabled = true, + InvalidAccount::DisabledAlias => input.aliases.as_mut().unwrap()[0].enabled = false, + InvalidAccount::RuntimeRoot | InvalidAccount::RuntimeAlias => { + let name = if matches!(invalid, InvalidAccount::RuntimeRoot) { + &input.name + } else { + &input.aliases.as_ref().unwrap()[0].name + }; + let config = app.active_provider.find_provider_config(name).unwrap(); + assert!(app + .active_provider + .observe_account(ProviderAccountObservation { + name: Arc::clone(name), + identity: config.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }) + .is_some()); + } + } + if !matches!(invalid, InvalidAccount::RuntimeRoot | InvalidAccount::RuntimeAlias) { + reload_inputs(app, vec![Arc::new(input.clone())]); + } + match invalid { + InvalidAccount::ExpiredRoot => ("overlap-input", HlsOriginAccountStatus::Expired), + InvalidAccount::ExpiredAlias => ("account-a", HlsOriginAccountStatus::Expired), + InvalidAccount::RemovedAlias | InvalidAccount::DisabledAlias => ("account-a", HlsOriginAccountStatus::Missing), + InvalidAccount::DisabledRoot | InvalidAccount::RuntimeRoot => { + ("overlap-input", HlsOriginAccountStatus::Blocked) + } + InvalidAccount::RuntimeAlias => ("account-a", HlsOriginAccountStatus::Blocked), + } +} + +#[tokio::test] +async fn handover_updates_all_viewers_without_followup_media_requests() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let proxy_id = session.read().await.proxy_session_id.clone(); + for (port, token) in [(12101, "viewer-a"), (12102, "viewer-b")] { + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &proxy_id, + token, + &test_fingerprint_with_addr(test_addr_with_port(port)), + token, + ) + .await; + } + let mut events = app.event_manager.get_event_channel(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .expect("replacement account publishes"); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "account-a"); + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 2); + assert!( + rows.iter().all(|row| row.provider.as_ref() == "account-a"), + "all viewer rows must change immediately: {rows:?}" + ); + let mut updated = Vec::new(); + while let Ok(event) = events.try_recv() { + if let shared::model::EventMessage::ActiveUser(shared::model::ActiveUserConnectionChange::Updated(row)) = event + { + updated.push(row); + } + } + assert_eq!(updated.len(), 2); + assert!(updated.iter().all(|row| row.provider.as_ref() == "account-a")); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); +} + +#[tokio::test] +async fn invalid_root_alias_matrix_rebinds_within_input_and_preserves_session() { + for invalid in [ + InvalidAccount::ExpiredRoot, + InvalidAccount::ExpiredAlias, + InvalidAccount::RemovedAlias, + InvalidAccount::DisabledRoot, + InvalidAccount::DisabledAlias, + InvalidAccount::RuntimeRoot, + InvalidAccount::RuntimeAlias, + ] { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let (old, expected_status) = invalidate_account(&app, &mut input, invalid); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", old, now).await; + let (key, proxy_id) = { + let session = session.read().await; + (session.key.clone(), session.proxy_session_id.clone()) + }; + let old_binding = session.read().await.origin_account_binding.clone().unwrap(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &proxy_id, + "matrix-viewer", + &test_fingerprint(), + "matrix-viewer", + ) + .await; + assert_eq!(hls_origin_account_status(&app.hls_ctx(), &old_binding), expected_status, "{invalid:?}"); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().unwrap(); + assert_eq!( + binding.account_name.as_ref(), + if old == "account-a" { "overlap-input" } else { "account-a" }, + "{invalid:?}" + ); + assert_eq!(binding.input_name, input.name); + assert_eq!(binding.generation, 1); + assert_eq!((&session.key, &session.proxy_session_id), (&key, &proxy_id)); + assert!(session.origin_account_manifest_pending.is_some()); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_provider_session_headers.is_empty()); + assert!(!prepared.fetch_url.contains(if old == "account-a" { "account-user" } else { "root-user" })); + let expected_provider = binding.account_name.clone(); + drop(session); + assert!( + app.active_users.panel_streams().await.iter().all(|row| row.provider == expected_provider), + "{invalid:?}" + ); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + } +} + +#[tokio::test] +async fn detached_failed_rebind_obeys_two_second_backoff_and_source_boundary() { + let mut input = overlap_provider_input(); + input.aliases = None; + let mut other = single_hls_provider_input("unrelated-source"); + other.id = 9; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", "removed-account", now).await; + let id = session.read().await.proxy_session_id.clone(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other.clone())]); + for timestamp in [now, now + 1_999] { + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + timestamp + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_rebind.consecutive_rebind_failures, 1); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + } + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other)]); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + now + 2_000, + ) + .await + .unwrap(); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().account_name, input.name); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + let reused_input = single_hls_provider_input("reused-input-id"); + reload_inputs(&app, vec![Arc::new(reused_input.clone())]); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &reused_input, + "http://account.example.com/index.m3u8", + "http://account.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + 4_000 + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().input_name, input.name); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + reload_inputs(&app, Vec::new()); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/index.m3u8", + "http://root.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + 4_000 + ) + .await + .is_err()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().input_name, input.name); +} + +#[tokio::test] +async fn concurrent_prepare_publishes_one_replacement() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..8 { + let app = Arc::clone(&app); + let input = input.clone(); + let session = Arc::clone(&session); + tasks.spawn(async move { + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +pub(super) fn enable_isolated_rebind_cache(app: &Arc, path: &std::path::Path) { + app.app_config.config.store(Arc::new(Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto { + cache_path: Some(path.to_string_lossy().into_owned()), + ..HlsCacheConfigDto::default() + }), + ..ReverseProxyConfigDto::default() + })), + ..Config::default() + })); +} + +pub(super) struct RebindOrigin { + pub(super) base_url: String, + pub(super) requests: Arc>>, + task: tokio::task::JoinHandle<()>, + block_old_media: Arc, + old_media_started: Arc, + release_old_media: Arc, +} +impl Drop for RebindOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +#[derive(Clone, Copy)] +enum RebindMediaLayout { + AccountPath, + SharedPath, +} + +async fn rebind_origin(extra_tags: &'static str, layout: RebindMediaLayout) -> RebindOrigin { + rebind_origin_with_window(extra_tags, layout, None, None).await +} + +pub(super) async fn resume_origin(head: Arc) -> RebindOrigin { + rebind_origin_with_window("", RebindMediaLayout::AccountPath, Some(head), None).await +} + +#[derive(Default)] +pub(super) struct ReducedRetryOriginControl { + pub(super) recovering: std::sync::atomic::AtomicBool, + pub(super) recovery_requests: std::sync::atomic::AtomicUsize, + pub(super) retry_started: tokio::sync::Notify, + pub(super) release_retry: tokio::sync::Notify, +} + +impl ReducedRetryOriginControl { + async fn manifest_body(&self) -> &'static [u8] { + if self.recovering.load(Ordering::Acquire) { + let n = self.recovery_requests.fetch_add(1, Ordering::AcqRel) + 1; + eprintln!("app recovery HTTP request={n} status=200"); + if n == 13 { + self.retry_started.notify_one(); + self.release_retry.notified().await; + } + if n <= 12 { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/reduced-retry/conflict.m3u8" + )) + } else { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/reduced-retry/repaired.m3u8" + )) + } + } else { + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/baseline.m3u8")) + } + } +} + +pub(super) async fn reduced_retry_origin(control: Arc) -> RebindOrigin { + rebind_origin_with_window("", RebindMediaLayout::AccountPath, None, Some(control)).await +} + +pub(super) fn resume_origin_media(path: &str) -> Vec { + // Valid TS, distinct from the enabled channel-unavailable asset. A null packet + // identifies the exact origin URI as well, preventing old cache hits from passing. + let mut media = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_connections_exhausted.ts")) + .to_vec(); + let mut marker = [0xff; 188]; + marker[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); + marker[4..4 + path.len()].copy_from_slice(path.as_bytes()); + media.extend_from_slice(&marker); + media +} + +async fn rebind_origin_with_window( + extra_tags: &'static str, + layout: RebindMediaLayout, + live_head: Option>, + recovery: Option>, +) -> RebindOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let block_old_media = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let old_media_started = Arc::new(tokio::sync::Notify::new()); + let release_old_media = Arc::new(tokio::sync::Semaphore::new(0)); + let (block, started, release) = + (Arc::clone(&block_old_media), Arc::clone(&old_media_started), Arc::clone(&release_old_media)); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let requests = Arc::clone(&recorded); + let live_head = live_head.clone(); + let recovery = recovery.clone(); + let (block, started, release) = (Arc::clone(&block), Arc::clone(&started), Arc::clone(&release)); + tokio::spawn(async move { + let mut buffer = [0; 4096]; + let count = socket.read(&mut buffer).await.unwrap(); + let request = String::from_utf8_lossy(&buffer[..count]).to_string(); + let path = request.lines().next().unwrap().split_whitespace().nth(1).unwrap().to_string(); + let alias_cookie = request.to_ascii_lowercase().contains("origin-account=alias"); + requests.lock().await.push(request); + if block.load(Ordering::Acquire) && path.contains("root-user") && path.contains(".ts") { + started.notify_one(); + release.acquire().await.unwrap().forget(); + } + let replacement = path.contains("account-user"); + let body = if let Some(control) = recovery { + // The playlists use virtual segment URIs; resume_origin_media supplies their TS bodies. + if path.ends_with("m3u8") { + control.manifest_body().await.to_vec() + } else { + resume_origin_media(&path) + } + } else if let Some(head) = live_head { + if path.ends_with("m3u8") { + super::manifest_entry_freshness::origin_playlist_with_duration(head.load(Ordering::Acquire), 11) + } else { + resume_origin_media(&path) + } + } else if path.ends_with("m3u8") { + let (head, count) = match (layout, replacement) { + (RebindMediaLayout::AccountPath, true) => (108, 8), + (RebindMediaLayout::SharedPath, true) => (102, 7), + (RebindMediaLayout::AccountPath | RebindMediaLayout::SharedPath, false) => (100, 8), + }; + let mut manifest = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + manifest.push_str(extra_tags); + for seq in head..head + count { + let uri = match layout { + RebindMediaLayout::AccountPath => format!("{seq}.ts"), + RebindMediaLayout::SharedPath => format!( + "/common/{seq}.ts?account={}", + if replacement { "account-user" } else { "root-user" } + ), + }; + writeln!(manifest, "#EXTINF:4,\n{uri}").unwrap(); + } + manifest.into_bytes() + } else if path == "/shared/key.bin" { + vec![if alias_cookie { 0x41 } else { 0x52 }; 16] + } else if path.contains(".ts") { + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/channel_unavailable.ts" + )) + .to_vec() + } else { + vec![0x47; 8192] + }; + let cookie = if replacement { "alias" } else { "root" }; + let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nSet-Cookie: origin-account={cookie}; Path=/\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(response.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + } + }); + RebindOrigin { base_url, requests, task, block_old_media, old_media_started, release_old_media } +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn segment_and_background_work_fetch_new_manifest_and_media_without_client_manifest() { + for (background, cross_host, provider_scheme) in + [(false, false, false), (true, false, false), (false, true, false), (true, false, true)] + { + let origin = + rebind_origin("", if cross_host { RebindMediaLayout::SharedPath } else { RebindMediaLayout::AccountPath }) + .await; + let cache_dir = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.url.clone_from(&origin.base_url); + input.aliases.as_mut().unwrap()[0].url = + if cross_host { origin.base_url.replace("127.0.0.1", "localhost") } else { origin.base_url.clone() }; + if provider_scheme { + let provider = Arc::new(crate::model::ConfigProvider::from(&shared::model::ConfigProviderDto { + name: "rebind-mirror".into(), + urls: vec![origin.base_url.clone().into()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + input.url = "provider://rebind-mirror".to_string(); + input.aliases.as_mut().unwrap()[0].url.clone_from(&input.url); + input.provider_configs = Some(vec![provider]); + } + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache_dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + enable_isolated_rebind_cache(&app, cache_dir.path()); + app.hls.proxy.update_config(&app.app_config).await; + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + let lease_uri = hls_proxy_uri(&app, &id.0, "000000.ts").await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", input.url); + let initial = prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + app.connection_manager.release_managed_provider_handle(initial.preacquired_origin_account_handle); + session.write().await.origin_account_manifest_pending = Some(0); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + // Wait for initial prefetches and retain one cached object as identity/cache evidence. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if session.read().await.active_segment_fetches == 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for seq in 0..8 { + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let file = HlsSegmentFile { proxy_seq: seq, extension: "ts".to_string() }; + let outcome = + app.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, &file, current_time_millis()).await; + assert_eq!(outcome, tuliprox_hls::api::SegmentDemandFetchOutcome::Ready); + } + session + .write() + .await + .render_and_store_manifest(current_time_millis()) + .expect("initial ready origin window is published"); + assert!(session.read().await.established_manifest_recovery_binding().is_some()); + let (key, cached, target_seq) = { + let mut s = session.write().await; + let cached = s + .segments + .iter() + .find_map(|(seq, entry)| matches!(entry.status, SegmentCacheStatus::Ready { .. }).then_some(*seq)); + let seq = *s.segments.keys().next_back().unwrap(); + let cache_key = s.segments[&seq].cache_key.clone(); + s.segments.get_mut(&seq).unwrap().status = SegmentCacheStatus::Discovered; + s.origin_provider_session_headers + .insert(header::COOKIE, HeaderValue::from_static("old-account=must-disappear")); + // Prior hard failure must not send recovery back to root credentials. + s.require_fresh_manifest_commit( + tuliprox_hls::api::HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + ); + s.origin_refresh.consecutive_failures = 1; + s.origin_refresh.next_fetch_allowed_at_ms = 0; + (s.key.clone(), cached, (seq, cache_key)) + }; + app.hls.proxy.segment_cache().delete(&target_seq.1).await.unwrap(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let boundary = origin.requests.lock().await.len(); + if background { + session.write().await.queue_segment_fetch_candidate( + target_seq.0, + tuliprox_hls::api::SegmentFetchPriority::Prefetch, + current_time_millis(), + ); + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + app.hls.proxy.segment_worker_pool().wake_scheduler(context, current_time_millis()).await; + } else { + let uri = lease_uri.replace("000000.ts", &format!("{:06}.ts", target_seq.0)); + let response = get_response(Arc::clone(&app), &uri, None).await; + assert!( + matches!(response.status(), StatusCode::OK | StatusCode::SERVICE_UNAVAILABLE), + "{}", + response.status() + ); + } + let replacement_ready = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let ready = session.read().await.segments.iter().find_map(|(seq, entry)| { + (matches!(entry.status, SegmentCacheStatus::Ready { .. }) + && entry + .origin_fetch_ref + .as_ref() + .is_some_and(|reference| reference.resolved_origin_url.contains("account-user"))) + .then_some(*seq) + }); + if let Some(seq) = ready { + break seq; + } + tokio::task::yield_now().await; + } + }) + .await; + let new_proxy_seq = match replacement_ready { + Ok(seq) => seq, + Err(error) => { + let s = session.read().await; + panic!("replacement media: background={background} host={cross_host} provider={provider_scheme}: {error}; pending={:?} refresh={:?} episode={:?} requests={:?}", s.origin_account_manifest_pending, s.origin_refresh, s.origin_control.acceptance_episode.as_ref().map(|episode| (&episode.state, episode.full_burst_completed)), origin.requests.lock().await.iter().map(|request| request.lines().next().unwrap()).collect::>()); + } + }; + let target = HlsSegmentFile { proxy_seq: new_proxy_seq, extension: "ts".to_string() }; + let context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let outcome = + app.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, &target, current_time_millis()).await; + assert_eq!( + outcome, + tuliprox_hls::api::SegmentDemandFetchOutcome::Ready, + "background={background} cross_host={cross_host} provider={provider_scheme}" + ); + let new_uri = lease_uri.replace("000000.ts", &format!("{new_proxy_seq:06}.ts")); + let response = get_response(Arc::clone(&app), &new_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_body(response).await.as_ref(), + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + ); + let s = session.read().await; + assert_eq!((&s.key, &s.proxy_session_id), (&key, &id)); + assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "account-a"); + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, 1); + assert!(s.origin_account_manifest_pending.is_none()); + if let Some(cached) = cached { + if cached != target_seq.0 { + assert!(matches!(s.segments[&cached].status, SegmentCacheStatus::Ready { .. })); + } + } + drop(s); + let requests = origin.requests.lock().await; + let replacement = &requests[boundary..]; + assert!( + replacement.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/12345.m3u8 ")), + "{replacement:?}" + ); + assert!( + replacement.iter().any(|request| request.lines().next().unwrap().contains("account-user") + && !request.lines().next().unwrap().contains("m3u8")), + "{replacement:?}" + ); + assert!( + replacement.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{replacement:?}" + ); + } +} + +#[tokio::test] +async fn independent_sessions_share_source_capacity_and_leave_other_source_unchanged() { + let mut input = overlap_provider_input(); + let mut alias = input.aliases.as_ref().unwrap()[0].clone(); + alias.id = 3; + alias.name = Arc::from("account-b"); + input.aliases.as_mut().unwrap().push(alias); + let mut other = single_hls_provider_input("other-source"); + other.id = 8; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let now = current_time_millis(); + let first = create_bound_hls_test_session(&app, &input, "first", &input.name, now).await; + let second = create_bound_hls_test_session(&app, &input, "second", &input.name, now).await; + let unaffected = create_bound_hls_test_session(&app, &other, "third", &other.name, now).await; + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone()), Arc::new(other.clone())]); + let mut handles = Vec::new(); + for session in [&first, &second] { + let id = session.read().await.proxy_session_id.clone(); + handles.push( + prepare_hls_origin_runtime( + &app, + session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(), + ); + } + assert_ne!( + first.read().await.origin_account_binding.as_ref().unwrap().account_name, + second.read().await.origin_account_binding.as_ref().unwrap().account_name + ); + assert_eq!(app.active_provider.get_provider_connections_count(), 2); + let unaffected = unaffected.read().await; + assert_eq!(unaffected.origin_account_binding.as_ref().unwrap().account_name, other.name); + assert_eq!(unaffected.origin_account_binding.as_ref().unwrap().generation, 0); + drop(handles); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn healthy_full_account_keeps_binding_and_cancelled_prepare_leaks_no_slot() { + let input = single_hls_provider_input("healthy"); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let occupied = app + .active_provider + .acquire_exact_connection_with_lease_for_session( + &input.name, + &test_addr(), + false, + 0, + ConnectionKind::Normal, + Some(crate::api::model::PlaybackLeaseRef::new("foreign-owner", crate::model::PlaybackKind::LiveHls)), + ) + .unwrap(); + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + assert!(prepared.preacquired_origin_account_handle.is_none()); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 0); + assert_eq!(session.read().await.origin_account_rebind.last_rebind_attempt_at_ms, None); + app.connection_manager.release_provider_handle(Some(occupied)); + let config = app.active_provider.find_provider_config(&input.name).unwrap(); + app.active_provider.observe_account(ProviderAccountObservation { + name: Arc::clone(&input.name), + identity: config.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }); + let lock = Arc::clone(&session.read().await.origin_account_prepare_lock); + let guard = lock.lock().await; + let task_session = Arc::clone(&session); + let task_app = Arc::clone(&app); + let task = tokio::spawn(async move { + prepare_hls_origin_runtime( + &task_app, + &task_session, + &input, + "http://account.example.com/index.m3u8", + "http://account.example.com/index.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Segment, + HlsOriginWorkClass::Demand, + now, + ) + .await + }); + tokio::task::yield_now().await; + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + drop(guard); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +async fn resource_rebind_fixture( + extra_tags: &'static str, +) -> (tempfile::TempDir, RebindOrigin, Arc, ConfigInput, HlsSessionHandle) { + let origin = rebind_origin(extra_tags, RebindMediaLayout::AccountPath).await; + let dir = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.url.clone_from(&origin.base_url); + input.aliases.as_mut().unwrap()[0].url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + enable_isolated_rebind_cache(&app, dir.path()); + app.hls.proxy.update_config(&app.app_config).await; + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + let _uri = hls_proxy_uri(&app, &id.0, "000000.ts").await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", input.url); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + drop(prepared); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + (dir, origin, app, input, session) +} + +#[tokio::test] +async fn map_worker_initiates_rebind_and_fetches_requalified_map() { + let (_dir, origin, app, _input, session) = resource_rebind_fixture("#EXT-X-MAP:URI=\"init.mp4\"\n").await; + let id = session.read().await.proxy_session_id.clone(); + let old_map = crate::api::model::MapEntry::new( + &id, + ProxyMapId(90), + OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: format!("{}/live/root-user/root-pass/old.mp4", origin.base_url), + byte_range: None, + }, + "mp4".to_string(), + ); + session.write().await.maps.insert(ProxyMapId(90), old_map); + let segment_context = + super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let context = tuliprox_hls::api::MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::clone(app.hls.proxy.segment_cache()), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: segment_context.origin_io, + }; + app.hls.proxy.map_worker_pool().wake_scheduler(context, current_time_millis()).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if session.read().await.maps.values().any(|map| { + map.origin_key.resolved_origin_uri.contains("account-user") + && matches!(map.status, MapCacheStatus::Ready { .. }) + }) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement manifest schedules its new MAP"); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + let requests = origin.requests.lock().await; + assert!( + requests.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/init.mp4 ")), + "{requests:?}" + ); + assert!( + requests.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{requests:?}" + ); +} + +#[tokio::test] +async fn transient_key_and_media_misses_rebind_before_reusing_any_old_url() { + for (kind, shared_uri) in [ + (TransientResourceKind::Key, false), + (TransientResourceKind::Segment, false), + (TransientResourceKind::Key, true), + ] { + let tags = if shared_uri { + "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"/shared/key.bin\"\n" + } else { + "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n" + }; + let (_dir, origin, app, _input, session) = resource_rebind_fixture(tags).await; + let now = current_time_millis(); + let resource = TransientResourceRef::new( + kind, + if shared_uri { + format!("{}/shared/key.bin", origin.base_url) + } else { + format!("{}/live/root-user/root-pass/old.bin", origin.base_url) + }, + &app.get_encrypt_secret(), + now, + 300_000, + Some("bin".to_string()), + ); + let id = session.read().await.proxy_session_id.clone(); + { + let mut s = session.write().await; + s.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + s.transient.upsert_resources([resource.clone()]); + s.origin_provider_session_headers + .insert(header::COOKIE, HeaderValue::from_static("old-account=must-disappear")); + } + let uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.bin", resource.id.0)).await; + let response = get_response(Arc::clone(&app), &uri, None).await; + assert!( + response.status() == StatusCode::SERVICE_UNAVAILABLE || response.status() == StatusCode::NOT_FOUND, + "{}", + response.status() + ); + assert!(response.headers().contains_key(header::RETRY_AFTER) || response.status() == StatusCode::NOT_FOUND); + let resource_id = resource.id.clone(); + let new_resource = session + .read() + .await + .transient + .resources + .values() + .find(|resource| { + resource.kind == kind + && (resource.resolved_origin_uri.contains("account-user") + || (shared_uri && resource.id != resource_id)) + }) + .cloned() + .expect("new manifest publishes replacement resources"); + let extension = new_resource.file_ext_hint.as_deref().unwrap(); + let uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.{extension}", new_resource.id.0)).await; + let response = get_response(Arc::clone(&app), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let expected: &[u8] = if kind == TransientResourceKind::Segment { + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + } else if shared_uri { + &[0x41; 16] + } else { + &[0x47; 8192] + }; + assert_eq!(response_body(response).await.as_ref(), expected); + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, 1); + let requests = origin.requests.lock().await; + assert!(requests.iter().any(|request| !request.lines().next().unwrap().contains("m3u8")), "{requests:?}"); + assert!( + requests.iter().all(|request| !request.contains("root-user") && !request.contains("old-account=")), + "{requests:?}" + ); + } +} + +#[tokio::test] +async fn abort_after_allocating_replacement_releases_its_handle() { + let mut input = overlap_provider_input(); + input.account_disabled = true; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let (allocated_tx, allocated_rx) = tokio::sync::oneshot::channel(); + let task_app = Arc::clone(&app); + let task = tokio::spawn(async move { + let id = session.read().await.proxy_session_id.clone(); + let prepared = prepare_hls_origin_runtime( + &task_app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(); + allocated_tx.send(()).unwrap(); + futures::future::pending::<()>().await; + drop(prepared); + }); + allocated_rx.await.unwrap(); + assert_eq!(app.active_provider.get_provider_connections_count(), 1); + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn late_origin_response_and_cleanup_cannot_overwrite_replacement() { + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n{}/live/root-user/root-pass/100.ts\n", + origin.base_url + )); + session.write().await.apply_origin_manifest(&manifest).unwrap(); + origin.block_old_media.store(true, Ordering::Release); + let context = super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let worker = Arc::clone(app.hls.proxy.segment_worker_pool()); + let old_request = tokio::spawn(async move { + worker + .demand_fetch_and_wait( + context, + &HlsSegmentFile { proxy_seq: 0, extension: "ts".to_string() }, + current_time_millis(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), origin.old_media_started.notified()) + .await + .expect("old request reaches controlled barrier"); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input)]); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + let replacement = session.read().await.origin_account_binding.clone().unwrap(); + let boundary = origin.requests.lock().await.len(); + origin.release_old_media.add_permits(1); + let _old_outcome = tokio::time::timeout(Duration::from_secs(5), old_request).await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let s = session.read().await; + if s.active_segment_fetches == 0 && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let s = session.read().await; + let binding = s.origin_account_binding.as_ref().unwrap(); + assert_eq!((&binding.account_name, binding.generation), (&replacement.account_name, replacement.generation)); + assert!(binding.is_active()); + assert!(s.segments.iter().any(|(seq, entry)| *seq > 0 && matches!(entry.status, SegmentCacheStatus::Ready { .. }))); + assert!(!matches!(s.segments[&0].status, SegmentCacheStatus::Ready { .. })); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); + assert!(origin.requests.lock().await[boundary..].iter().all(|request| !request.contains("root-user"))); +} + +struct BlockAccountBeforeAcquire { + delegate: Arc, + app: std::sync::Weak, + account: Arc, + calls: AtomicUsize, +} +impl std::fmt::Debug for BlockAccountBeforeAcquire { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockAccountBeforeAcquire").finish_non_exhaustive() + } +} +impl tuliprox_hls::api::HlsOriginAccountRuntime for BlockAccountBeforeAcquire { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tuliprox_hls::api::HlsPreparedOriginAccount, + tuliprox_hls::api::HlsBoundAccountAcquireErrorKind, + >, + > + Send + + 'a, + >, + > { + Box::pin(async move { + let prepared = self.delegate.prepare(session, work, class).await?; + if self.calls.fetch_add(1, Ordering::AcqRel) == 1 { + let app = self.app.upgrade().unwrap(); + let provider = app.active_provider.find_provider_config(&self.account).unwrap(); + app.active_provider.observe_account(ProviderAccountObservation { + name: Arc::clone(&self.account), + identity: provider.account_identity(), + status: Some(shared::model::ProxyUserStatus::Banned), + exp_date: None, + }); + } + Ok(prepared) + }) + } +} + +#[tokio::test] +async fn account_blocked_between_prepare_and_acquire_still_rebinds_from_media_work() { + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + let mut config = app.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().max_concurrent_segment_fetches_global = 1; + app.app_config.config.store(Arc::new(config)); + app.hls.proxy.update_config(&app.app_config).await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let delegate = session.read().await.origin_account_runtime.clone().unwrap(); + session.write().await.origin_account_runtime = Some(Arc::new(BlockAccountBeforeAcquire { + delegate, + app: Arc::downgrade(&app), + account: Arc::clone(&input.name), + calls: AtomicUsize::new(0), + })); + let manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n{}/live/root-user/root-pass/100.ts\n", + origin.base_url + )); + session.write().await.apply_origin_manifest(&manifest).unwrap(); + let context = super::super::build_hls_segment_fetch_context(&app, &session, None, &test_fingerprint(), None).await; + let _old_outcome = app + .hls + .proxy + .segment_worker_pool() + .demand_fetch_and_wait( + context, + &HlsSegmentFile { proxy_seq: 0, extension: "ts".to_string() }, + current_time_millis(), + ) + .await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let s = session.read().await; + if s.origin_account_binding.as_ref().is_some_and(|binding| binding.account_name.as_ref() == "account-a") + && s.segments.values().any(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. })) + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("confirmed invalidity at acquisition initiates replacement"); + let requests = origin.requests.lock().await; + assert!(requests + .iter() + .any(|request| request.contains("account-user") && request.lines().next().unwrap().contains(".ts"))); + assert!(requests.iter().all(|request| !request.contains("root-user"))); +} + +#[tokio::test] +async fn identical_key_uri_caches_distinct_account_bytes_and_keeps_published_old_key() { + let (_dir, origin, app, mut input, session) = + resource_rebind_fixture("#EXT-X-KEY:METHOD=AES-128,URI=\"/shared/key.bin\"\n").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let old_key = TransientResourceRef::new( + TransientResourceKind::Key, + format!("{}/shared/key.bin", origin.base_url), + &app.get_encrypt_secret(), + now, + 300_000, + Some("bin".to_string()), + ); + let id = { + let mut current = session.write().await; + current.transient.upsert_resources([old_key.clone()]); + current.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("origin-account=root")); + current.proxy_session_id.clone() + }; + let old_uri = hls_proxy_uri(&app, &id.0, &format!("r/{}.bin", old_key.id.0)).await; + let response = get_response(Arc::clone(&app), &old_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x52; 16]); + assert!(session + .read() + .await + .transient + .ready_key_object_valid_until_ms(&id, &old_key.id, "bin", current_time_millis()) + .is_some()); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input)]); + let runtime = session.read().await.origin_account_runtime.clone().unwrap(); + runtime.prepare(&session, HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand).await.unwrap(); + let new_key = session + .read() + .await + .transient + .resources + .values() + .find(|resource| { + resource.kind == TransientResourceKind::Key + && resource.id != old_key.id + && resource.resolved_origin_uri == old_key.resolved_origin_uri + }) + .cloned() + .expect("new account publishes a separate key identity for the same URI"); + let extension = new_key.file_ext_hint.as_deref().unwrap(); + let prefix = old_uri.rsplit_once("/r/").unwrap().0; + let new_uri = format!("{prefix}/r/{}.{extension}", new_key.id.0); + let lease_id = old_uri.split('/').nth(5).unwrap(); + super::support::publish_test_transient_resource_membership(&app, &id.0, lease_id, &format!("{old_uri}\n{new_uri}")) + .await; + let response = get_response(Arc::clone(&app), &new_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x41; 16]); + let request_count = + origin.requests.lock().await.iter().filter(|request| request.starts_with("GET /shared/key.bin ")).count(); + let response = get_response(Arc::clone(&app), &old_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await.as_ref(), &[0x52; 16]); + assert_eq!( + origin.requests.lock().await.iter().filter(|request| request.starts_with("GET /shared/key.bin ")).count(), + request_count, + "old published key remains a cache hit" + ); +} + +#[tokio::test] +async fn manifest_request_built_after_a_handoff_uses_the_snapshot_entry_and_reprepares_it() { + use super::super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}; + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + session.write().await.replace_origin_account_binding(None); + let id = session.read().await.proxy_session_id.clone(); + let root_url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + &root_url, + &root_url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(); + // Preserve the exact result across a controlled asynchronous boundary. + let (continue_tx, continue_rx) = tokio::sync::oneshot::channel(); + let task_app = Arc::clone(&app); + let task_session = Arc::clone(&session); + let task = tokio::spawn(async move { + continue_rx.await.unwrap(); + let mut io = super::provider_admission::test_hls_origin_io_context(&task_app); + io.account_runtime = task_session.read().await.origin_account_runtime.clone(); + if let Some(handle) = prepared.preacquired_origin_account_handle { + io = io.with_managed_provider_handle(handle); + } + let request = hls_origin_refresh_request( + &task_app, + &task_session, + tuliprox_hls::api::HlsPreparedOriginAccount { + entry: tuliprox_hls::api::LiveHlsOriginEntry::parse(&prepared.fetch_url).unwrap(), + snapshot: prepared.snapshot, + }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: tuliprox_hls::api::HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: tuliprox_hls::api::HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: task_app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + tuliprox_hls::api::trigger_origin_refresh_sync(request).await + }); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + drop( + prepare_hls_origin_runtime( + &app, + &session, + &input, + &root_url, + &root_url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(), + ); + continue_tx.send(()).unwrap(); + assert!(task.await.unwrap()); + let requests = origin.requests.lock().await; + assert!(requests.iter().any(|request| request.starts_with("GET /live/account-user/account-pass/12345.m3u8 "))); + assert!(requests.iter().all(|request| !request.contains("root-user")), "{requests:?}"); +} + +struct PausedOriginPrepare { + delegate: Arc, + prepared: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} + +impl std::fmt::Debug for PausedOriginPrepare { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("PausedOriginPrepare").finish_non_exhaustive() + } +} + +impl tuliprox_hls::api::HlsOriginAccountRuntime for PausedOriginPrepare { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result< + tuliprox_hls::api::HlsPreparedOriginAccount, + tuliprox_hls::api::HlsBoundAccountAcquireErrorKind, + >, + > + Send + + 'a, + >, + > { + Box::pin(async move { + let prepared = self.delegate.prepare(session, work, class).await?; + self.prepared.notify_one(); + self.release.acquire().await.unwrap().forget(); + Ok(prepared) + }) + } +} + +#[tokio::test] +async fn runtime_prepare_result_superseded_before_refresh_start_cannot_authorize_its_old_entry() { + use super::super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}; + let (_dir, origin, app, mut input, session) = resource_rebind_fixture("").await; + input.account_disabled = false; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let runtime = Arc::new(PausedOriginPrepare { + delegate: session.read().await.origin_account_runtime.clone().unwrap(), + prepared: tokio::sync::Notify::new(), + release: tokio::sync::Semaphore::new(0), + }); + let mut io = super::provider_admission::test_hls_origin_io_context(&app); + io.account_runtime = Some(runtime.clone()); + let entry = tuliprox_hls::api::LiveHlsOriginEntry::parse(&format!( + "{}/live/root-user/root-pass/12345.m3u8", + origin.base_url + )) + .unwrap(); + let request = hls_origin_refresh_request( + &app, + &session, + tuliprox_hls::api::HlsPreparedOriginAccount { + entry, + snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot::from_session(&*session.read().await), + }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: tuliprox_hls::api::HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: tuliprox_hls::api::HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + rewrite_secret: app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + // Force a new prepare and pause its actual app-adapter result before the start CAS. + let mut request = request; + request.origin_io.as_mut().unwrap().prepared_account = None; + let refresh = tokio::spawn(tuliprox_hls::api::maybe_trigger_origin_refresh_with_outcome(request)); + tokio::time::timeout(Duration::from_secs(5), runtime.prepared.notified()).await.unwrap(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let id = session.read().await.proxy_session_id.clone(); + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + drop( + prepare_hls_origin_runtime( + &app, + &session, + &input, + &url, + &url, + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + current_time_millis(), + ) + .await + .unwrap(), + ); + runtime.release.add_permits(1); + assert_eq!(refresh.await.unwrap(), tuliprox_hls::api::HlsOriginRefreshTriggerOutcome::SessionUnavailable); + assert!(origin.requests.lock().await.is_empty(), "superseded runtime result sends no origin request"); + assert!(!session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn failed_rebind_does_not_publish_replacement_provider() { + let mut input = overlap_provider_input(); + input.aliases = None; + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let before = app.active_users.panel_streams().await; + let mut events = app.event_manager.get_event_channel(); + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + assert!(prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now + ) + .await + .is_err()); + assert_eq!(app.active_users.panel_streams().await, before); + while let Ok(event) = events.try_recv() { + assert!(!matches!( + event, + shared::model::EventMessage::ActiveUser(shared::model::ActiveUserConnectionChange::Updated(_)) + )); + } +} + +#[tokio::test] +async fn older_handover_cannot_overwrite_newer_provider() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let original = session.read().await.origin_account_binding.clone().unwrap(); + let publish = |name: &'static str, generation| { + HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from(name), + original.session_owner.clone(), + generation, + now, + ) + }; + let binding_b = publish("account-b", 1); + let snapshot_b = publish_hls_origin_account_with_status(&app, &session, |current| { + current.handoff_origin_account(binding_b); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + let ready = Arc::new(tokio::sync::Notify::new()); + let resume = Arc::new(tokio::sync::Notify::new()); + let old = { + let app = app.clone(); + let session = session.clone(); + let ready = ready.clone(); + let resume = resume.clone(); + tokio::spawn(async move { + ready.notify_one(); + resume.notified().await; + publish_hls_origin_account_with_status(&app, &session, |_| Ok(snapshot_b)).await + }) + }; + ready.notified().await; + let binding_c = publish("account-c", 2); + publish_hls_origin_account_with_status(&app, &session, |current| { + current.handoff_origin_account(binding_c); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + let mut events = app.event_manager.get_event_channel(); + resume.notify_one(); + assert!(old.await.unwrap().is_err()); + assert_eq!(app.active_users.panel_streams().await[0].provider.as_ref(), "account-c"); + assert!(events.try_recv().is_err()); +} + +#[tokio::test] +async fn stale_media_registration_cannot_restore_previous_provider() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "viewer", + &test_fingerprint(), + "viewer", + ) + .await; + let gate = session.read().await.origin_account_prepare_lock.clone(); + let held = gate.lock().await; + let fp = test_fingerprint(); + let headers = HeaderMap::new(); + let access = test_hls_access_context_with(id.clone(), HlsAccessLeaseId("viewer".into()), "viewer", fp.key.clone()); + let mut registration = + Box::pin(super::super::ensure_hls_cache_stream_registered(&app, &fp, &headers, &access, &session)); + assert!(futures::poll!(registration.as_mut()).is_pending()); + publish_hls_origin_account_with_status(&app, &session, |current| { + let old = current.origin_account_binding.as_ref().unwrap(); + let next = HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from("account-b"), + old.session_owner.clone(), + old.generation + 1, + current_time_millis(), + ); + current.handoff_origin_account(next); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + drop(held); + assert_eq!(registration.await.unwrap().provider.as_ref(), "account-b"); + assert_eq!(app.active_users.panel_streams().await[0].provider.as_ref(), "account-b"); +} + +#[tokio::test] +async fn initiating_viewer_cancellation_does_not_lose_handover_updates() { + let mut input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let now = current_time_millis(); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, now).await; + let id = session.read().await.proxy_session_id.clone(); + for (port, token) in [(12201, "initiator"), (12202, "remaining")] { + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + token, + &test_fingerprint_with_addr(test_addr_with_port(port)), + token, + ) + .await; + } + input.account_disabled = true; + reload_inputs(&app, vec![Arc::new(input.clone())]); + let published = Arc::new(tokio::sync::Notify::new()); + let request = { + let app = app.clone(); + let session = session.clone(); + let id = id.clone(); + let published = published.clone(); + tokio::spawn(async move { + let _prepared = prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now, + ) + .await + .unwrap(); + published.notify_one(); + std::future::pending::<()>().await; + }) + }; + published.notified().await; + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + assert!(app.active_users.panel_streams().await.iter().all(|row| row.provider.as_ref() == "account-a")); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "late-viewer", + &test_fingerprint_with_addr(test_addr_with_port(12203)), + "late-viewer", + ) + .await; + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 3); + assert!(rows.iter().all(|row| row.provider.as_ref() == "account-a")); +} + +#[tokio::test] +async fn late_join_and_status_snapshot_use_current_origin_account() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let id = session.read().await.proxy_session_id.clone(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "existing", + &test_fingerprint(), + "existing", + ) + .await; + publish_hls_origin_account_with_status(&app, &session, |current| { + let previous = current.origin_account_binding.as_ref().unwrap(); + let next = HlsOriginAccountBinding::rebound( + input.name.clone(), + Arc::from("account-b"), + previous.session_owner.clone(), + previous.generation + 1, + current_time_millis(), + ); + current.handoff_origin_account(next); + Ok(HlsOriginAccountSnapshot::from_session(current)) + }) + .await + .unwrap(); + super::observability::register_hls_cache_stream_for_stats_test( + &app, + &session, + &id, + "late", + &test_fingerprint_with_addr(test_addr_with_port(12301)), + "late", + ) + .await; + let rows = app.active_users.panel_streams().await; + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.provider.as_ref() == "account-b")); +} + +#[tokio::test] +async fn handover_waits_for_viewer_projection_without_holding_hls_write_lock() { + use super::super::origin_account_runtime::publish_hls_origin_account_with_status; + use tuliprox_hls::api::HlsOriginAccountSnapshot; + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", &input.name, current_time_millis()).await; + let shared_id = super::super::hls_cache_shared_stream_id(&session.read().await.proxy_session_id); + let held = app.active_users.lock_shared_stream_provider_projection(shared_id).await; + let mut publish = Box::pin(publish_hls_origin_account_with_status(&app, &session, |current| { + Ok(HlsOriginAccountSnapshot::from_session(current)) + })); + assert!(futures::poll!(publish.as_mut()).is_pending()); + assert!(session.try_write().is_ok(), "manager wait must never retain an HLS lock"); + drop(held); + publish.await.unwrap(); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs b/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs index 2ac34e07c..01164019e 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/owner_token.rs @@ -491,7 +491,7 @@ async fn tokens_without_entry_kind_keep_bad_request_without_session() { let response = fixture.token_request(&client("10.0.0.1", 50_121), &token, HeaderMap::new()).await; assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - assert!(fixture.origin.requests().is_empty()); + assert_eq!(fixture.origin.requests().len(), 0); } #[tokio::test] @@ -507,7 +507,7 @@ async fn shared_hls_cache_on_never_recreates_from_token() { ); let response = fixture.token_request(&client("10.0.0.1", 50_131), &token, HeaderMap::new()).await; assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert!(fixture.origin.requests().is_empty()); + assert_eq!(fixture.origin.requests().len(), 0); assert!(fixture.app_state.active_users.get_and_update_user_session("hls-user", hint).await.is_none()); } diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs index 2650ff9bd..85bb46bfd 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs @@ -21,6 +21,9 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_hls_origin_io_context( reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, } } diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs index 80f0951cf..4d48a48fb 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs @@ -156,8 +156,8 @@ fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() { #[tokio::test] async fn hls_account_overlap_reclaim_preempts_speculative_session() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await; let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await; let winner_proxy_session_id = winner.read().await.proxy_session_id.clone(); @@ -175,7 +175,9 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() { } let loser_generation = loser.read().await.activity.origin_work_generation; - super::super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; + winner.write().await.mark_authorized_media_access(10_000); + winner.write().await.activity.last_delivered_media_at_ms = Some(10_000); + drop(prepare_overlap_origin(&app_state, &input, &winner, 10_000).await); assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await); assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await); @@ -191,8 +193,8 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() { #[tokio::test] async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await; let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await; let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone(); @@ -210,7 +212,9 @@ async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { } let displaced_generation = displaced.read().await.activity.origin_work_generation; - super::super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await; + promoted.write().await.mark_authorized_media_access(20_001); + promoted.write().await.activity.last_delivered_media_at_ms = Some(20_001); + drop(prepare_overlap_origin(&app_state, &input, &promoted, 20_001).await); let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); assert!(matches!( @@ -262,8 +266,7 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( .await .expect("interactive work should use soft-active overlap before grace"); - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); + let binding = new_session.read().await.origin_account_binding.clone().expect("speculative binding is published"); assert_eq!(binding.account_name, input.name); assert!(matches!( &binding.binding_mode, @@ -273,11 +276,15 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work( } if displaced_proxy_session_id == &old_proxy_session_id )); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::Available(_)) )); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -324,14 +331,16 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { .await .expect("normal HLS policy should preempt active soft HLS binding"); - let new_binding = prepared_origin - .origin_account_binding_to_store - .as_ref() - .expect("preempting session should receive active binding"); + let new_binding = + normal_session.read().await.origin_account_binding.clone().expect("preempting session receives active binding"); assert_eq!(new_binding.account_name, input.name); assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::Available(_)) )); let soft_session = soft_session.read().await; @@ -345,7 +354,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { )); drop(soft_session); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -437,7 +446,6 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal let result = super::super::prepare_hls_origin_policy_preempt_runtime( &app_state, - &soft_session, &input, "http://account.example.com/live/account-user/account-pass/soft.m3u8", "http://account.example.com/live/account-user/account-pass/soft.m3u8", @@ -496,12 +504,18 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai .expect("interactive work can use grace when normal acquire and overlap fail"); assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + prepared_origin + .preacquired_origin_account_handle + .as_ref() + .and_then(tuliprox_session::ManagedProviderHandle::handle) + .map(|handle| &handle.allocation), Some(super::super::ProviderAllocation::GracePeriod(_)) )); assert_eq!( - prepared_origin - .origin_account_binding_to_store + session + .read() + .await + .origin_account_binding .as_ref() .expect("grace binding should still bind the selected account") .account_name @@ -509,7 +523,7 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai input.name.as_ref() ); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); app_state.connection_manager.release_provider_handle(Some(occupied)); } @@ -715,3 +729,40 @@ fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_ super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted ); } + +async fn prepare_overlap_origin( + app: &Arc, + input: &ConfigInput, + session: &tuliprox_hls::api::HlsSessionHandle, + now_ms: u64, +) -> super::super::session::PreparedHlsOriginRuntime { + let id = session.read().await.proxy_session_id.clone(); + super::super::prepare_hls_origin_runtime( + app, + session, + input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + now_ms, + ) + .await + .unwrap() +} + +#[tokio::test] +async fn healthy_prepare_detaches_an_unprotected_expired_binding_before_reacquisition() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "expired-protection", "account-a", 1_000).await; + let generation = session.read().await.activity.origin_work_generation; + drop(prepare_overlap_origin(&app, &input, &session, 10_000).await); + let current = session.read().await; + assert!(current.activity.origin_work_generation > generation); + assert!(current.origin_account_binding.as_ref().unwrap().generation > 0); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs index e65886aa3..6cd8428ba 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs @@ -21,6 +21,33 @@ use http_body_util::BodyExt; use shared::model::{HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster}; use std::{sync::Arc, time::Duration}; +#[tokio::test] +async fn origin_resource_no_account_available_includes_retry_after() { + let app_state = test_app_state(); + let access_context = test_hls_access_context( + ProxySessionId("shared-hls-session".to_string()), + HlsAccessLeaseId("access-lease".to_string()), + ); + + for reason in [ + super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, + super::super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, + ] { + let response = super::super::hls_origin_runtime_resource_failure_response( + &app_state, + &access_context, + super::super::HlsOriginRuntimeAcquireError::NoAccountAvailable { reason }, + ); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{reason:?}"); + assert_eq!( + response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), + Some("2"), + "{reason:?}" + ); + } +} + #[tokio::test] async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() { let input = single_hls_provider_input("known-account"); @@ -52,7 +79,7 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho } let old_generation = session.read().await.activity.origin_work_generation; - super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await; + super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 6_001).await; { let session = session.read().await; @@ -77,17 +104,16 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho 0, super::super::HlsOriginWorkKind::Manifest, super::super::HlsOriginWorkClass::ManifestInteractive, - 4_100, + 6_100, ) .await .expect("authorized origin work can reacquire a detached binding"); - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); + let binding = session.read().await.origin_account_binding.clone().expect("new binding is published atomically"); assert_eq!(binding.account_name.as_ref(), "account-a"); assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle); } #[tokio::test] @@ -122,6 +148,70 @@ async fn hls_account_binding_without_media_activity_expires_after_startup_window assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); } +#[tokio::test] +async fn fresh_hls_binding_has_bounded_startup_protection_despite_old_media_delivery() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await; + { + let mut s = session.write().await; + s.target_duration = Some(11); + s.activity.last_authorized_media_at_ms = Some(70_000); + s.activity.last_delivered_media_at_ms = Some(70_000); + } + let proxy_id = session.read().await.proxy_session_id.clone(); + for now in [200_001, 204_999, 205_000] { + let prepared = super::super::prepare_hls_origin_runtime( + &app, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Resource, + super::super::HlsOriginWorkClass::Demand, + now, + ) + .await + .unwrap(); + let s = session.read().await; + let binding = s.origin_account_binding.as_ref().unwrap(); + assert!(binding.is_active()); + assert_eq!(binding.pinned_at_ms, 200_000, "preparing the same binding cannot extend startup protection"); + assert_eq!(binding.generation, 0); + assert_eq!(s.activity.last_delivered_media_at_ms, Some(70_000)); + drop(s); + app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle); + } + super::super::detach_unprotected_hls_origin_account_bindings(&app, 205_001).await; + assert!( + session.read().await.origin_account_binding.as_ref().unwrap().is_detached(), + "no media progress: releasable after five seconds" + ); +} + +#[tokio::test] +async fn hls_binding_expired_startup_protects_running_origin_work() { + let input = overlap_provider_input(); + let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await; + let generation = { + let mut s = session.write().await; + s.target_duration = Some(11); + s.activity.last_authorized_media_at_ms = Some(70_000); + s.activity.last_delivered_media_at_ms = Some(70_000); + s.start_origin_work() + }; + super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_000).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_active()); + assert!(session.write().await.finish_origin_work(generation)); + super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_001).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached()); +} + #[tokio::test] async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { let app_state = test_app_state(); @@ -145,6 +235,7 @@ async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_d host_local_index: 123, }, proxy_seq: 123, + media_start_ms: 0, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs index eeeb814e6..e33d8f0ad 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs @@ -95,7 +95,7 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. })); } assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs new file mode 100644 index 000000000..c8cc4608e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs @@ -0,0 +1,287 @@ +use super::{ + super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions}, + manifest_entry_freshness::open_manifest_for_new_lease, + origin_account_rebind::{ + enable_isolated_rebind_cache, reduced_retry_origin, resume_origin_media, ReducedRetryOriginControl, + }, + *, +}; +use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + trigger_origin_refresh_sync, HlsManifestCommitRequirement, HlsOriginAccountSnapshot, HlsPreparedOriginAccount, +}; + +const WAIT: Duration = Duration::from_secs(15); + +async fn ready(app: &Arc, session: &HlsSessionHandle, sequences: &[u64]) { + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if sequences.iter().all(|seq| { + s.segments.get(seq).is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. })) + }) && s.active_segment_fetches == 0 + && !s.origin_refresh.in_flight + && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("autonomous segment work and singleflight complete"); + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +async fn deliver(app: &Arc, session: &HlsSessionHandle, manifest: &str, expected: &[u64]) { + assert!(!manifest.contains("/cvs/hls/")); + assert!(!manifest.contains("#EXT-X-ENDLIST")); + for sequence in expected { + let (proxy_seq, url) = { + let s = session.read().await; + let (seq, entry) = + s.segments.iter().find(|(_, entry)| entry.origin_key.host_local_sequence == *sequence).unwrap(); + (*seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()) + }; + let uri = manifest + .lines() + .find(|line| line.ends_with(&format!("/{proxy_seq:06}.ts"))) + .expect("segment URI is published by Tuliprox"); + let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await; + assert_eq!(response.status(), StatusCode::OK); + let bytes = response_body(response).await; + let path = reqwest::Url::parse(&url).unwrap().path().to_string(); + assert_eq!(bytes.as_ref(), resume_origin_media(&path)); + eprintln!("app delivered resource={path} bytes={}", bytes.len()); + } +} + +#[expect(clippy::too_many_lines, reason = "Keep the ordered end-to-end recovery scenario and its assertions together.")] +async fn repaired_reduced_retry(mode: HlsStartupMode) { + let control = Arc::new(ReducedRetryOriginControl::default()); + let origin = reduced_retry_origin(Arc::clone(&control)).await; + let cache = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.name = Arc::from("account-a"); + input.priority = 0; + input.url.clone_from(&origin.base_url); + let b = &mut input.aliases.as_mut().unwrap()[0]; + b.name = Arc::from("account-b"); + b.priority = 10; + b.url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)), + vec![Arc::new(input.clone())], + ); + let available_b = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-b"), + &ProxySessionId("availability-check".into()), + current_time_millis(), + ); + assert_eq!( + tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b), + tuliprox_hls::api::HlsOriginAccountStatus::Known + ); + enable_isolated_rebind_cache(&app, cache.path()); + let mut config = app.app_config.config.load_full().as_ref().clone(); + let hls = config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap(); + hls.startup = (&HlsStartupConfigDto { mode, ..Default::default() }).into(); + hls.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + app.app_config.config.store(Arc::new(config)); + enable_channel_unavailable_custom_response(&app); + app.hls.proxy.update_config(&app.app_config).await; + create_active_hls_user_session(&app).await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let lease = "reduced-retry-baseline"; + let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await; + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await; + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &HlsAccessLeaseId(lease.to_string()), + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("complete baseline is published to the playback lease"); + assert_eq!(response.status(), StatusCode::OK); + let first = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + deliver(&app, &session, &first, &[1000]).await; + tokio::time::timeout(WAIT, async { + let proxy_id = session.read().await.proxy_session_id.clone(); + loop { + let snapshot = app + .hls + .proxy + .access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis()) + .await + .unwrap(); + if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("baseline body completion is recorded before recovery pressure is evaluated"); + let (binding, baseline, commits, proxy_id) = { + let s = session.read().await; + assert!(s.activity.last_authorized_media_at_ms.is_some()); + assert!(s.activity.last_delivered_media_at_ms.is_some()); + assert!(s.established_manifest_recovery_binding().is_some()); + ( + s.origin_account_binding.clone().unwrap(), + s.origin_to_proxy.clone(), + s.origin_manifest_commits(), + s.proxy_session_id.clone(), + ) + }; + assert_eq!(binding.account_name.as_ref(), "account-a"); + eprintln!( + "{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}", + binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0 + ); + session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure; + let directive = + match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id) + .await + { + tuliprox_hls::api::HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, + other => panic!("recovery pressure must evaluate: {other:?}"), + }; + eprintln!("app baseline directive={directive:?}"); + assert!(directive.recovery_pressure_guard.is_some(), "directive has current lease/session evidence"); + assert_eq!(directive.trigger, tuliprox_hls::HlsManifestAcceptanceTrigger::RecoveryRequired); + let mut io = super::provider_admission::test_hls_origin_io_context(&app); + io.account_runtime = session.read().await.origin_account_runtime.clone(); + let snapshot = HlsOriginAccountSnapshot::from_session(&*session.read().await); + let mut request = hls_origin_refresh_request( + &app, + &session, + HlsPreparedOriginAccount { entry: LiveHlsOriginEntry::parse(&url).unwrap(), snapshot }, + HlsOriginRefreshOptions { + headers: HeaderMap::new(), + origin_io: io, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + acceptance_directive: directive, + access_lease_id: Some(HlsAccessLeaseId(lease.to_string())), + rewrite_secret: app.get_encrypt_secret().to_vec(), + now_ms: current_time_millis(), + }, + ); + request.now_ms = session.read().await.origin_refresh.next_fetch_allowed_at_ms.max(current_time_millis()); + control.recovering.store(true, Ordering::Release); + let refresh = tokio::spawn(trigger_origin_refresh_sync(request)); + tokio::time::timeout(WAIT, control.retry_started.notified()).await.expect("reduced HTTP attempt 13 starts"); + let episode_before = { + let s = session.read().await; + assert_eq!(s.origin_to_proxy, baseline); + assert_eq!(s.origin_manifest_commits(), commits, "first actual conflict commit did not mutate the timeline"); + let episode = s.origin_control.acceptance_episode.clone().unwrap(); + assert_eq!(episode.full_bursts_completed, 1); + assert!(episode.exhaustion_reason().is_some()); + assert!(episode.deterministic_conflict_receipt().is_none()); + eprintln!( + "{mode:?} before response 13: generation={} full_bursts={} reason={:?} timing={:?}", + episode.generation.0, + episode.full_bursts_completed, + episode.exhaustion_reason(), + episode.timing() + ); + episode + }; + let prepare_gate = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test(); + control.release_retry.notify_one(); + tokio::time::timeout(WAIT, prepare_gate.entered.notified()) + .await + .expect("repaired reduced retry commits and schedules media"); + let scheduled = { + let s = session.read().await; + let mut resources = s + .segment_prefetch_queue + .proxy_seqs() + .iter() + .map(|seq| { + let entry = &s.segments[seq]; + ( + *seq, + entry.origin_key.host_local_sequence, + entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone(), + ) + }) + .collect::>(); + resources.sort(); + assert_eq!(resources.iter().map(|(_, seq, _)| *seq).collect::>(), vec![1006, 1007, 1008]); + assert_eq!(s.origin_manifest_commits(), commits + 1); + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation); + assert_eq!(s.origin_control.acceptance_generation, episode_before.generation); + assert!( + s.origin_control.acceptance_episode.is_none(), + "successful refresh bookkeeping consumes the completed episode" + ); + eprintln!("{mode:?} committed generation={} scheduled={resources:?}", episode_before.generation.0); + resources + }; + prepare_gate.prepare.notify_one(); + tokio::time::timeout(WAIT, prepare_gate.prepared.notified()).await.unwrap(); + prepare_gate.proceed.notify_one(); + let seqs = scheduled.iter().map(|(seq, _, _)| *seq).collect::>(); + ready(&app, &session, &seqs).await; + tokio::time::timeout(WAIT, refresh).await.unwrap().unwrap(); + assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13); + { + let requests = origin.requests.lock().await; + for (_, _, url) in &scheduled { + let path = reqwest::Url::parse(url).unwrap().path().to_string(); + let count = requests.iter().filter(|request| request.starts_with(&format!("GET {path} "))).count(); + assert!(count > 0, "queued resource was fetched before any client demand"); + eprintln!("{mode:?} autonomous origin GET {path} requests={count} status=Ready"); + } + assert!(requests.iter().all(|request| !request.contains("account-user")), "B remains unused"); + } + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &HlsAccessLeaseId(lease.to_string()), + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("repaired live manifest is published"); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + deliver(&app, &session, &body, &[1006, 1007, 1008]).await; + let s = session.read().await; + assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation); + assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name, binding.account_name); + assert_eq!(s.origin_account_binding.as_ref().unwrap().pinned_at_ms, binding.pinned_at_ms); + assert_eq!(s.origin_account_binding.as_ref().unwrap().provider_binding_tag, binding.provider_binding_tag); + assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13); + eprintln!( + "{mode:?} complete: recovery_requests=13 account={} binding_generation={} origin_work=0 singleflight=false", + binding.account_name, binding.generation + ); + assert_eq!(s.proxy_session_id, proxy_id); + assert!(!s.origin_refresh.in_flight); + assert_eq!(s.active_segment_fetches, 0); + assert_eq!(s.activity.active_origin_work_count.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn hls_reduced_retry_delivers_repaired_origin_media_optimistic() { + repaired_reduced_retry(HlsStartupMode::Optimistic).await; +} + +#[tokio::test] +async fn hls_reduced_retry_delivers_repaired_origin_media_conservative() { + repaired_reduced_retry(HlsStartupMode::Conservative).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs new file mode 100644 index 000000000..7cabb3ffc --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs @@ -0,0 +1,294 @@ +use super::{ + manifest_entry_freshness::open_manifest_for_new_lease, + origin_account_rebind::{enable_isolated_rebind_cache, resume_origin, resume_origin_media}, + *, +}; +use shared::model::{HlsStartupConfigDto, HlsStartupMode}; +use std::sync::atomic::AtomicU64; +use tuliprox_core::utils::current_time_millis; + +const PAUSE_MS: u64 = 130_000; +const WAIT: Duration = Duration::from_secs(5); + +async fn settle_origin(session: &HlsSessionHandle) { + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if !s.origin_refresh.in_flight + && s.active_segment_fetches == 0 + && s.activity.active_origin_work_count.load(Ordering::Acquire) == 0 + { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("origin work settles before playback is stopped"); +} + +/// Model wall-clock absence without erasing actual media delivery, or relying on +/// Tokio's clock (account preparation and HTTP serving also read `SystemTime`). +async fn pause_playback(app: &Arc, session: &HlsSessionHandle, lease: &str) { + settle_origin(session).await; + assert!(app + .hls + .proxy + .segment_worker_pool() + .access_leases() + .write() + .await + .remove_access_lease(&HlsAccessLeaseId(lease.to_string())) + .is_some()); + app.active_users.release_session_streams_and_counted_reservation("hls-user", "hls-session-token").await; + let now = current_time_millis(); + let proxy_id = { + let mut s = session.write().await; + let age = |time: &mut Option| *time = time.map(|time| time.saturating_sub(PAUSE_MS)); + age(&mut s.activity.last_authorized_manifest_at_ms); + age(&mut s.activity.last_authorized_media_at_ms); + age(&mut s.activity.last_delivered_media_at_ms); + s.last_client_access_at_ms = s.last_client_access_at_ms.saturating_sub(PAUSE_MS); + let binding = s.origin_account_binding.as_mut().unwrap(); + binding.pinned_at_ms = binding.pinned_at_ms.saturating_sub(PAUSE_MS); + age(&mut binding.last_origin_io_at_ms); + age(&mut binding.last_reservation_refresh_at_ms); + age(&mut s.origin_refresh.last_fetch_started_at_ms); + age(&mut s.origin_refresh.last_fetch_finished_at_ms); + age(&mut s.origin_refresh.last_success_at_ms); + age(&mut s.origin_refresh.last_error_at_ms); + s.origin_refresh.next_fetch_allowed_at_ms = s.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(PAUSE_MS); + if let Some(rendered) = s.last_rendered_manifest.as_mut() { + rendered.rendered_at_ms = rendered.rendered_at_ms.saturating_sub(PAUSE_MS); + rendered.valid_until_ms = rendered.valid_until_ms.saturating_sub(PAUSE_MS); + } + assert!(s.activity.last_delivered_media_at_ms.is_some_and(|last| now - last >= PAUSE_MS)); + assert!(!s.can_expire_idle_session(now, 300_000), "retained session is inside its 300-second timeout"); + s.proxy_session_id.clone() + }; + app.hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app.active_users, + &app.active_provider, + session, + &proxy_id, + now, + ) + .await; + super::super::detach_unprotected_hls_origin_account_bindings(app, now).await; + assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached()); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "pause releases provider slots"); + for account in ["account-a", "account-b"] { + assert_eq!( + app.active_provider.provider_lease_usage(&Arc::from(account)).total(), + 0, + "no logical slot remains for {account}" + ); + } +} + +fn segment_uris(manifest: &str) -> Vec<&str> { + assert!(manifest.starts_with("#EXTM3U"), "live playlist: {manifest}"); + assert!(!manifest.contains("/cvs/hls/"), "replacement video must never satisfy resume: {manifest}"); + assert!(!manifest.contains("#EXT-X-ENDLIST"), "resume remains live"); + let uris = manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::>(); + assert!(!uris.is_empty(), "live media playlist must contain media"); + uris +} + +async fn deliver_origin_segment(app: &Arc, session: &HlsSessionHandle, manifest: &str, head: u64) { + let uri = segment_uris(manifest)[0]; + let seq = uri.rsplit('/').next().unwrap().trim_end_matches(".ts").parse::().unwrap(); + let origin_url = { + let s = session.read().await; + let segment = &s.segments[&seq]; + assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window"); + segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone() + }; + let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string(); + let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = response_body(response).await; + assert_eq!(body.as_ref(), resume_origin_media(&path), "client receives the exact current origin object"); + let s = session.read().await; + assert!(s.activity.last_authorized_media_at_ms.is_some()); + assert!(s.activity.last_delivered_media_at_ms.is_some(), "fully consume the Tuliprox response before pausing"); + eprintln!( + "delivered origin={path} bytes={} authorized={:?} delivered={:?}", + body.len(), + s.activity.last_authorized_media_at_ms, + s.activity.last_delivered_media_at_ms + ); +} + +#[expect(clippy::too_many_lines, reason = "Keep the ordered pause/resume scenario and its assertions together.")] +async fn same_account_resume(mode: HlsStartupMode, resumes: usize) { + let head = Arc::new(AtomicU64::new(1113)); + let origin = resume_origin(Arc::clone(&head)).await; + let cache = tempfile::tempdir().unwrap(); + let mut input = overlap_provider_input(); + input.name = Arc::from("account-a"); + input.priority = 0; + input.url.clone_from(&origin.base_url); + let b = &mut input.aliases.as_mut().unwrap()[0]; + b.name = Arc::from("account-b"); + b.priority = 10; + b.url.clone_from(&origin.base_url); + let app = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)), + vec![Arc::new(input.clone())], + ); + let available_b = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-b"), + &ProxySessionId("availability-check".into()), + current_time_millis(), + ); + assert_eq!( + tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b), + tuliprox_hls::api::HlsOriginAccountStatus::Known + ); + enable_isolated_rebind_cache(&app, cache.path()); + let mut config = app.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().startup = + (&HlsStartupConfigDto { mode, ..HlsStartupConfigDto::default() }).into(); + app.app_config.config.store(Arc::new(config)); + enable_channel_unavailable_custom_response(&app); + app.hls.proxy.update_config(&app.app_config).await; + assert_eq!(app.hls.proxy.session_idle_timeout_ms(), 300_000); + create_active_hls_user_session(&app).await; + let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url); + let key = super::super::build_hls_origin_source(&input, "12345").session_key(); + let mut lease = "resume-initial".to_string(); + let first = open_manifest_for_new_lease(&app, &input, &url, &lease).await; + let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + deliver_origin_segment(&app, &session, &first, 1113).await; + settle_origin(&session).await; + let proxy_id = session.read().await.proxy_session_id.clone(); + + for cycle in 0..resumes { + let old = session.read().await.origin_account_binding.clone().unwrap(); + assert_eq!(old.account_name.as_ref(), "account-a"); + pause_playback(&app, &session, &lease).await; + let current_head = 1126 + u64::try_from(cycle).unwrap() * 13; + head.store(current_head, Ordering::Release); + lease = format!("resume-{cycle}"); + create_active_hls_user_session(&app).await; + let boundary = origin.requests.lock().await.len(); + let started_at = current_time_millis(); + let barrier = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test(); + let (task_app, task_input, task_url, task_lease) = + (Arc::clone(&app), input.clone(), url.clone(), lease.clone()); + let opening = + tokio::spawn( + async move { open_manifest_for_new_lease(&task_app, &task_input, &task_url, &task_lease).await }, + ); + tokio::time::timeout(WAIT, barrier.entered.notified()).await.expect("manifest commit reaches scheduler"); + let retained = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); + assert!(Arc::ptr_eq(&session, &retained), "same shared session allocation"); + let (rebound, scheduled, delivered_before_prepare) = { + let s = session.read().await; + assert_eq!(s.key, key); + assert_eq!(s.proxy_session_id, proxy_id); + assert_eq!(s.target_duration, Some(11)); + let rebound = s.origin_account_binding.clone().unwrap(); + assert_eq!(rebound.account_name, old.account_name, "B is available; A is actually rebound"); + assert!(rebound.is_active()); + assert!(rebound.generation > old.generation); + assert!(rebound.pinned_at_ms >= started_at); + let mut scheduled = s + .segment_prefetch_queue + .proxy_seqs() + .into_iter() + .map(|seq| { + let entry = &s.segments[&seq]; + assert!((current_head..current_head + 6).contains(&entry.origin_key.host_local_sequence)); + (seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()) + }) + .collect::>(); + scheduled.sort(); + assert!(!scheduled.is_empty(), "capture jobs immediately after commit, before preparation"); + eprintln!("{mode:?} cycle={cycle} shared={key:?} proxy={proxy_id:?} account={} binding={}->{} pinned={} scheduled={scheduled:?}", rebound.account_name, old.generation, rebound.generation, rebound.pinned_at_ms); + (rebound, scheduled, s.activity.last_delivered_media_at_ms) + }; + barrier.prepare.notify_one(); + tokio::time::timeout(WAIT, barrier.prepared.notified()).await.expect("resource preparation completes"); + eprintln!( + "requests through prepare={:?}", + origin.requests.lock().await[boundary..] + .iter() + .map(|request| request.lines().next().unwrap()) + .collect::>() + ); + { + let s = session.read().await; + eprintln!( + "after prepare: binding={:?} queue={:?} states={:?}", + s.origin_account_binding, + s.segment_prefetch_queue.proxy_seqs(), + scheduled.iter().map(|(seq, _)| (*seq, &s.segments[seq].status)).collect::>() + ); + assert!( + scheduled.iter().all(|(seq, _)| matches!( + s.segments[seq].status, + SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::Ready { .. } + )), + "manifest segment jobs were lost during resource preparation" + ); + let current = s.origin_account_binding.as_ref().unwrap(); + assert_eq!(current.generation, rebound.generation, "fresh binding stays stable into media work"); + assert_eq!(current.pinned_at_ms, rebound.pinned_at_ms); + assert_eq!( + s.activity.last_delivered_media_at_ms, delivered_before_prepare, + "manifest fetch is not media delivery" + ); + } + barrier.proceed.notify_one(); + // No demand fetch or test-side requeue: every original job must progress autonomously. + tokio::time::timeout(WAIT, async { + loop { + let s = session.read().await; + if scheduled.iter().all(|(seq, _)| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("all originally scheduled resources reach Ready"); + let manifest = tokio::time::timeout(WAIT, opening).await.unwrap().unwrap(); + let requests = origin.requests.lock().await[boundary..] + .iter() + .map(|request| request.lines().next().unwrap().to_string()) + .collect::>(); + eprintln!("{mode:?} cycle={cycle} origin requests={requests:?}"); + for (_, url) in &scheduled { + let path = reqwest::Url::parse(url).unwrap().path().to_string(); + assert!( + requests.iter().any(|line| line == &format!("GET {path} HTTP/1.1")), + "scheduled origin request: {path}" + ); + } + assert!(requests.iter().all(|line| !line.contains("account-user")), "B remains unused"); + deliver_origin_segment(&app, &session, &manifest, current_head).await; + assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, rebound.generation); + } + pause_playback(&app, &session, &lease).await; + assert!(origin.requests.lock().await.iter().all(|request| !request.contains("account-user"))); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "no provider slot leak after repeated pauses"); +} + +#[tokio::test] +async fn hls_same_account_resume_keeps_scheduled_segments() { + same_account_resume(HlsStartupMode::Optimistic, 2).await; +} + +#[tokio::test] +async fn hls_same_account_resume_keeps_scheduled_segments_conservative() { + same_account_resume(HlsStartupMode::Conservative, 2).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs index 605d51ce1..09859fa69 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs @@ -346,6 +346,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_segment_entry( host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: proxy_seq * 4_000, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs index 7199acc23..9a7cc551f 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs @@ -94,6 +94,8 @@ pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body_rang SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, ); entry.duration_ms = 10_000; + entry.media_start_ms = proxy_seq * 10_000; + session.next_media_start_ms = session.next_media_start_ms.max(entry.media_start_ms + entry.duration_ms); session.segments.insert(proxy_seq, entry); } session.advance_media_readiness_generation(); diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs index cf1b8a190..b0231d0f9 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs @@ -225,6 +225,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_runtime_policy_t HlsLeasePlaybackMode::Live => "live", HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", + HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response", HlsLeasePlaybackMode::Ended => "ended", }); panic!( diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs index e29da4016..73182959d 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs @@ -265,7 +265,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_session_last_media_at pub(in crate::api::endpoints::hls_api::tests) async fn assert_no_hls_cache_stream_registered( app_state: &Arc, ) { - assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!(app_state.active_users.active_streams().await.len(), 0); } pub(in crate::api::endpoints::hls_api::tests) async fn access_lease_session_token( diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs index 5090b3914..e259a478b 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs @@ -230,11 +230,16 @@ pub(in crate::api::endpoints::hls_api::tests) async fn run_recovery_outage( let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence"); assert!(episode.full_burst_completed); assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - last_episode_generation = Some(episode.generation.0); + if let Some(generation) = last_episode_generation { + assert_eq!(episode.generation.0, generation, "unchanged failures share the initial full-burst receipt"); + } else { + last_episode_generation = Some(episode.generation.0); + } } - assert!( - fixture.origin.manifest_request_count().saturating_sub(requests_before) - >= plan.total_candidates().saturating_mul(3) + assert_eq!( + fixture.origin.manifest_request_count().saturating_sub(requests_before), + plan.total_candidates() + fixture.refresh.retry_policy.attempt_count().saturating_sub(1) + 2, + "one configured full burst, its bounded reduced retries, then one sample per due refresh" ); let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0); let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await; @@ -794,9 +799,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1); let commit_guard_seq = transition_buffer_seq.saturating_add(1); + let base_tail = session.segments.get(&fixture.base_manifest.last_proxy_seq).expect("live base tail"); + let mut media_start_ms = base_tail.media_start_ms.saturating_add(base_tail.duration_ms); for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) { + // Shortened test buffers must remain contiguous on the session media clock. + segment.media_start_ms = media_start_ms; if segment.proxy_seq == transition_buffer_seq { segment.duration_ms = 1_000; } else if segment.proxy_seq == commit_guard_seq { @@ -804,6 +813,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr } else { segment.status = SegmentCacheStatus::Expired; } + media_start_ms = media_start_ms.saturating_add(segment.duration_ms); } session.advance_media_readiness_generation(); } @@ -915,7 +925,11 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_ .playback_cursor .first_segment_completed_at_ms .expect("measured lease playback start") - .saturating_add(fixture.base_manifest.playlist_duration_ms); + // Enter the acquisition window while the live base and its key are still usable. + .saturating_add( + fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms), + ); + let first = commit_terminal_tail_if_lease_reserve_requires_cutover( &fixture.app_state.hls_ctx(), &fixture.session, @@ -1309,6 +1323,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn assert_publication_late_t assert!(body.contains("#EXT-X-ENDLIST")); assert!(body.contains("/terminal/")); } + HlsLeasePlaybackMode::TerminalCustomResponse(_) => { + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(body.contains("/cvs/hls/")); + assert!(!body.contains("/hls/shared/live/")); + } HlsLeasePlaybackMode::TerminalUnavailable { .. } => { assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } @@ -1352,6 +1373,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_refresh_reque transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id, disabled_headers: None, diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs index 2d7657594..7072a7945 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs @@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path() } #[tokio::test] -async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { +async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() { let fixture = prepared_terminal_cutover_fixture().await; let directive = terminal_cutover_acceptance_directive(&fixture).await; let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await; @@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { } #[tokio::test] -async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() { +async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() { let temp_dir = tempfile::tempdir().expect("tempdir"); let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); enable_channel_unavailable_custom_response(&app_state); @@ -147,16 +147,30 @@ async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_s let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.status(), StatusCode::OK); assert!(!response.headers().contains_key(header::LOCATION)); - assert!(response_body(response).await.is_empty()); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest"); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + assert!(body.contains("/cvs/hls/")); + assert!(!body.contains("#EXT-X-MAP")); + assert!(!body.contains("/hls/shared/live/")); let lease = app_state .hls .proxy .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) .await - .expect("failed-closed lease snapshot"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + .expect("late custom lease snapshot"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + assert_eq!(repeated.status(), StatusCode::OK); + assert_eq!(response_body(repeated).await, body.as_bytes()); + let repeated_lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("stable terminal custom lease"); + assert_eq!(repeated_lease.playback_mode, lease.playback_mode); let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); let terminal_response = get_response(app_state, &terminal_uri, None).await; assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); diff --git a/backend/app/src/api/endpoints/hls_terminal_response.rs b/backend/app/src/api/endpoints/hls_terminal_response.rs index 4080c2846..a53838a7e 100644 --- a/backend/app/src/api/endpoints/hls_terminal_response.rs +++ b/backend/app/src/api/endpoints/hls_terminal_response.rs @@ -44,6 +44,7 @@ pub(super) async fn hls_manifest_terminal_preflight( ) -> HlsManifestTerminalPreflight { match &lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return HlsManifestTerminalPreflight::ServeCommittedPlayback, HlsLeasePlaybackMode::Live => {} @@ -183,6 +184,13 @@ pub(super) fn hls_terminal_playback_response( }; Some(hls_response(body.to_owned())) } + HlsLeasePlaybackMode::TerminalCustomResponse(plan) => { + Some(if plan.matches_route(proxy_session_id, access_lease_id) { + hls_response(plan.manifest_body().to_owned()) + } else { + StatusCode::NOT_FOUND.into_response() + }) + } HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(StatusCode::SERVICE_UNAVAILABLE.into_response()), HlsLeasePlaybackMode::Live => None, HlsLeasePlaybackMode::Ended => Some(StatusCode::NOT_FOUND.into_response()), diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index f035bf822..7512dd183 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -1186,11 +1186,11 @@ mod tests { assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].protocol.is_empty()); - assert!(api_proxy.server[0].host.is_empty()); + assert_eq!(api_proxy.server[0].protocol.len(), 0); + assert_eq!(api_proxy.server[0].host.len(), 0); assert!(api_proxy.server[0].port.is_none()); - assert!(api_proxy.server[0].timezone.is_empty()); - assert!(api_proxy.server[0].message.is_empty()); + assert_eq!(api_proxy.server[0].timezone.len(), 0); + assert_eq!(api_proxy.server[0].message.len(), 0); assert!(api_proxy.server[0].path.is_none()); assert_eq!(api_proxy.user.len(), 1); assert!(!api_proxy.use_user_db); @@ -1206,8 +1206,8 @@ mod tests { assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].host.is_empty()); - assert!(api_proxy.user.is_empty()); + assert_eq!(api_proxy.server[0].host.len(), 0); + assert_eq!(api_proxy.user.len(), 0); } #[test] @@ -1217,8 +1217,8 @@ mod tests { filter_api_proxy_by_permissions(&mut api_proxy, permissions); - assert!(api_proxy.server.is_empty()); - assert!(api_proxy.user.is_empty()); + assert_eq!(api_proxy.server.len(), 0); + assert_eq!(api_proxy.user.len(), 0); assert!(!api_proxy.use_user_db); assert_eq!(api_proxy.auth_error_status, 403); } @@ -1270,20 +1270,20 @@ mod tests { filter_app_config_by_permissions(&mut app_config, Some(permissions)); assert_eq!(app_config.config.storage_dir, None); - assert!(app_config.sources.inputs.is_empty()); + assert_eq!(app_config.sources.inputs.len(), 0); assert!(app_config.sources.provider.is_none()); assert!(app_config.sources.templates.is_none()); assert!(app_config.mappings.is_none()); assert!(app_config.templates.is_none()); assert_eq!(app_config.sources.sources.len(), 1); - assert!(app_config.sources.sources[0].inputs.is_empty()); + assert_eq!(app_config.sources.sources[0].inputs.len(), 0); assert_eq!(app_config.sources.sources[0].targets.len(), 1); let target = &app_config.sources.sources[0].targets[0]; assert_eq!(target.name, "Default"); assert_eq!(target.id, 1); assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled); - assert!(target.output.is_empty()); + assert_eq!(target.output.len(), 0); assert!(target.curation.is_none(), "identity-only access must not disclose discovery credentials"); assert!(target.filter.is_empty()); assert!(target.options.is_none()); @@ -1296,7 +1296,7 @@ mod tests { let api_proxy = app_config.api_proxy.expect("api proxy should remain present"); assert_eq!(api_proxy.server.len(), 1); assert_eq!(api_proxy.server[0].name, "main"); - assert!(api_proxy.server[0].host.is_empty()); + assert_eq!(api_proxy.server[0].host.len(), 0); assert_eq!(api_proxy.user.len(), 1); } diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs index 0bdc1c88c..57f7bf774 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs @@ -344,8 +344,8 @@ async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests( assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); } else { assert!(!queued.targets.enabled); - assert!(queued.targets.targets.is_empty()); - assert!(queued.targets.target_names.is_empty()); + assert_eq!(queued.targets.targets.len(), 0); + assert_eq!(queued.targets.target_names.len(), 0); } } } @@ -582,7 +582,7 @@ async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_c assert_eq!(status.inputs[0].clusters[2].timestamp, 202); assert_eq!(status.inputs[1].input_id, 8); assert_eq!(status.inputs[1].last_update, None); - assert!(status.inputs[1].clusters.is_empty()); + assert_eq!(status.inputs[1].clusters.len(), 0); assert_eq!(status.inputs[2].input_id, 9); assert_eq!(status.inputs[2].last_update, Some(404)); assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed"); diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs index c53329ee0..a6ee09985 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs @@ -50,7 +50,7 @@ async fn playlist_update_status_reload_returns_active_bus_facts_without_persisti PlaylistUpdateState::Failure, ))); let Json(status) = super::super::playlist_update_status(State(app)).await.unwrap(); - assert!(status.active_updates.is_empty()); + assert_eq!(status.active_updates.len(), 0); assert_eq!( std::fs::read(path.join("status.json")).unwrap(), before, diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index 9422ee5fb..053e9f138 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -747,7 +747,7 @@ fn m3u_timeshift_resolver_returns_resolved_url_for_flussonic_template() { .expect("M3U input handled by bridge"); assert_eq!(resolved.url, "http://provider.example/channel/video-1704067200-3600.m3u8"); - assert!(!resolved.discriminator.is_empty()); + assert_ne!(resolved.discriminator.len(), 0); } #[test] diff --git a/backend/app/tests/hls_origin_account_handover.rs b/backend/app/tests/hls_origin_account_handover.rs new file mode 100644 index 000000000..d95001c08 --- /dev/null +++ b/backend/app/tests/hls_origin_account_handover.rs @@ -0,0 +1,353 @@ +//! Real-process regression: an origin account publication must reach every `WebUI` socket immediately. +use axum::{ + extract::{OriginalUri, State}, + response::IntoResponse, + Router, +}; +use futures::{SinkExt, StreamExt}; +use shared::model::{ActiveUserConnectionChange, ProtocolMessage, SourcesConfigDto, StreamInfo, PROTOCOL_VERSION}; +use std::{ + fmt::Write, + path::{Path, PathBuf}, + process::Stdio, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::Notify, +}; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; + +type Socket = WebSocketStream>; + +#[derive(Clone)] +struct Origin { + catalog: String, + manifests: Arc, + segments: Arc, + proceed: Arc, +} + +async fn origin(State(state): State, OriginalUri(uri): OriginalUri) -> axum::response::Response { + let path = uri.path(); + if path == "/__continue" { + state.proceed.notify_one(); + return "continued".into_response(); + } + if path.ends_with("catalog.m3u") { + return ([("content-type", "audio/x-mpegurl")], state.catalog).into_response(); + } + if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) { + state.manifests.fetch_add(1, Ordering::AcqRel); + return ([("content-type", "application/vnd.apple.mpegurl")], "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n100.ts\n#EXTINF:4,\n101.ts\n#EXTINF:4,\n102.ts\n#EXTINF:4,\n103.ts\n#EXTINF:4,\n104.ts\n#EXTINF:4,\n105.ts\n").into_response(); + } + if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) { + state.segments.fetch_add(1, Ordering::AcqRel); + return ( + [("content-type", "video/mp2t")], + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ) + .into_response(); + } + axum::http::StatusCode::NOT_FOUND.into_response() +} + +struct Fixture { + directory: tempfile::TempDir, + base: String, + source: PathBuf, + process: tokio::process::Child, + origins: Vec>, + b: Origin, + proceed: Arc, + control: String, +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.process.start_kill(); + for origin in &self.origins { + origin.abort(); + } + } +} + +impl Fixture { + async fn start() -> Self { + let a = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let b = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let a_addr = format!("localhost:{}", a.local_addr().unwrap().port()); + let b_addr = b.local_addr().unwrap(); + let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = reserved.local_addr().unwrap().port(); + drop(reserved); + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + for name in ["web", "storage", "backup"] { + std::fs::create_dir(root.join(name)).unwrap(); + } + let frontend = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist"); + let web = if frontend.join("index.html").exists() { frontend } else { root.join("web") }; + let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: true\n user_ui_enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n", web.display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display()); + let source_text = format!("inputs:\n - name: account-a\n type: m3u\n url: http://{a_addr}/catalog.m3u\n max_connections: 8\n aliases:\n - name: account-b\n url: http://{b_addr}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [account-a]\n targets:\n - name: handover\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n"); + let mut users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Handover test\nuser:\n - target: handover\n credentials:\n"); + for user in ["alice", "bob", "late"] { + write!(users, " - username: {user}\n password: test-password\n proxy: reverse\n server: local\n status: Active\n").unwrap(); + } + let config_path = root.join("config.yml"); + let source = root.join("source.yml"); + let proxy = root.join("api-proxy.yml"); + std::fs::write(&config_path, config).unwrap(); + std::fs::write(&source, source_text).unwrap(); + std::fs::write(&proxy, users).unwrap(); + let proceed = Arc::new(Notify::new()); + let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"handover\" group-title=\"Test\",Handover Channel\nhttp://{a_addr}/live/index.m3u8\n"); + let state = |catalog: String| Origin { + catalog, + manifests: Arc::new(AtomicUsize::new(0)), + segments: Arc::new(AtomicUsize::new(0)), + proceed: proceed.clone(), + }; + let a_state = state(catalog.clone()); + let b_state = state(catalog.replace(&a_addr, &b_addr.to_string())); + let mut origins = Vec::new(); + for (listener, state) in [(a, a_state), (b, b_state.clone())] { + origins.push(tokio::spawn(async move { + axum::serve(listener, Router::new().fallback(origin).with_state(state)).await.unwrap(); + })); + } + let log = std::fs::File::create(root.join("sut.log")).unwrap(); + let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox")) + .args(["-s", "-c"]) + .arg(config_path) + .arg("-i") + .arg(&source) + .arg("-a") + .arg(proxy) + .current_dir(root) + .stdout(Stdio::from(log.try_clone().unwrap())) + .stderr(Stdio::from(log)) + .kill_on_drop(true) + .spawn() + .unwrap(); + Self { + directory, + base: format!("http://127.0.0.1:{port}"), + source, + process, + origins, + b: b_state, + proceed, + control: format!("http://{a_addr}/__continue"), + } + } + + async fn playback(&mut self, client: &reqwest::Client, username: &str) -> String { + let url = format!("{}/m3u?username={username}&password=test-password", self.base); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + assert!( + self.process.try_wait().unwrap().is_none(), + "SUT exited: {}", + std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() + ); + if let Ok(response) = client.get(&url).send().await { + if let Ok(body) = response.text().await { + if let Some(url) = body.lines().find(|line| line.starts_with("http")) { + return url.to_string(); + } + } + } + assert!( + tokio::time::Instant::now() < deadline, + "playlist unavailable: {}", + std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + async fn wait_for_browser_handover(&self, viewer_clients: &[reqwest::Client], viewer_media: &[reqwest::Url]) { + eprintln!( + "BROWSER_READY web={} control={} fixture={}", + self.base, + self.control, + self.directory.path().display() + ); + // Keep the two manual test players running while the browser loads. Stop all + // media requests before publishing B; the browser must then update from events alone. + let mut playback = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + () = self.proceed.notified() => break, + _ = playback.tick() => { + for (client, media) in viewer_clients.iter().zip(viewer_media) { + let response = client.get(media.clone()).send().await.unwrap(); + assert!(response.status().is_success()); + response.bytes().await.unwrap(); + } + } + } + } + } +} + +async fn socket(base: &str) -> Socket { + let (mut socket, _) = connect_async(format!("{}/ws", base.replace("http://", "ws://"))).await.unwrap(); + socket.send(Message::Binary(ProtocolMessage::Version(PROTOCOL_VERSION).to_bytes().unwrap())).await.unwrap(); + assert!( + matches!(socket.next().await.unwrap().unwrap(), Message::Binary(bytes) if bytes.as_ref() == [PROTOCOL_VERSION]) + ); + socket +} + +async fn frame(socket: &mut Socket) -> ProtocolMessage { + loop { + let next = tokio::time::timeout(Duration::from_secs(15), socket.next()) + .await + .expect("WebUI frame arrives without polling") + .unwrap() + .unwrap(); + if let Message::Binary(bytes) = next { + return ProtocolMessage::from_bytes(bytes).unwrap(); + } + } +} + +async fn snapshot(socket: &mut Socket) -> Vec { + socket.send(Message::Binary(ProtocolMessage::StatusRequest(String::new()).to_bytes().unwrap())).await.unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if let ProtocolMessage::StatusResponse(status) = + tokio::time::timeout_at(deadline, frame(socket)).await.expect("full status snapshot arrives") + { + return status.active_user_streams; + } + } +} + +async fn changed_viewers(socket: &mut Socket, before: &[StreamInfo]) { + let mut seen = std::collections::HashSet::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while seen.len() < before.len() { + match tokio::time::timeout_at(deadline, frame(socket)).await.expect("all viewers receive Updated frames") { + ProtocolMessage::ActiveUserResponse(ActiveUserConnectionChange::Updated(row)) + if row.provider.as_ref() == "account-b" => + { + let mut expected = before + .iter() + .find(|old| old.uid == row.uid && old.addr == row.addr) + .expect("same viewer row") + .clone(); + expected.provider = Arc::from("account-b"); + assert_eq!(row, expected, "handover changes only the displayed provider"); + seen.insert(row.uid); + } + ProtocolMessage::StatusResponse(_) => panic!("handover must arrive as incremental Updated frames"), + _ => {} + } + } +} + +#[tokio::test] +async fn hls_origin_account_handover() { + let mut fixture = Fixture::start().await; + let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap(); + let mut canonical = Vec::new(); + let mut viewer_clients = Vec::new(); + let mut viewer_media = Vec::new(); + for viewer in ["alice", "bob"] { + let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap(); + let mut entry = fixture.playback(&client, viewer).await; + let mut manifest = None; + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while tokio::time::Instant::now() < deadline { + let response = client.get(&entry).send().await.unwrap(); + if response.status().is_success() { + let base = response.url().clone(); + let body = response.text().await.unwrap(); + let segments = body + .lines() + .filter(|line| !line.starts_with('#') && !line.is_empty()) + .map(|line| base.join(line).unwrap()) + .collect::>(); + if let Some(segment) = segments.first() { + if body.contains("#EXT-X-STREAM-INF:") { + entry = segment.to_string(); + continue; + } + let media = client.get(segment.clone()).send().await.unwrap(); + assert!(media.status().is_success()); + assert!( + !media.bytes().await.unwrap().is_empty(), + "viewer actually consumes media before the segment gap" + ); + viewer_media.push(segment.clone()); + manifest = Some(base); + break; + } + } + tokio::task::yield_now().await; + } + canonical.push(manifest.expect("viewer has a canonical playable manifest")); + viewer_clients.push(client); + } + let mut ui_a = socket(&fixture.base).await; + let mut ui_b = socket(&fixture.base).await; + let mut before = snapshot(&mut ui_a).await; + assert_eq!( + before.len(), + 2, + "canonical={canonical:?} SUT log: {}", + std::fs::read_to_string(fixture.directory.path().join("sut.log")).unwrap() + ); + assert!(before.iter().all(|row| row.provider.as_ref() == "account-a")); + let mut before_b = snapshot(&mut ui_b).await; + assert_eq!(before_b.len(), 2); + let manual = std::env::var_os("TULIPROX_HANDOVER_BROWSER_CHECK").is_some(); + if manual { + fixture.wait_for_browser_handover(&viewer_clients, &viewer_media).await; + // Playback above may advance timestamps; capture the exact pre-publication rows. + before = snapshot(&mut ui_a).await; + before_b = snapshot(&mut ui_b).await; + assert_eq!(before.len(), 2); + assert_eq!(before_b.len(), 2); + } + let mut sources: SourcesConfigDto = + serde_saphyr::from_str(&std::fs::read_to_string(&fixture.source).unwrap()).unwrap(); + sources.inputs[0].account_disabled = true; + let config = client.get(format!("{}/api/v1/config", fixture.base)).send().await.unwrap(); + let revision = + config.headers().get(shared::utils::HEADER_CONFIG_SOURCES_REVISION).expect("source revision").clone(); + let response = client + .post(format!("{}/api/v1/config/sources", fixture.base)) + .header(reqwest::header::IF_MATCH, revision) + .json(&sources) + .send() + .await + .unwrap(); + assert!(response.status().is_success(), "source reload: {}", response.text().await.unwrap()); + // Only a manifest refresh triggers the handover; neither viewer requests another segment or status. + let response = client.get(canonical[0].clone()).send().await.unwrap(); + assert!(response.status().is_success() || response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE); + changed_viewers(&mut ui_a, &before).await; + changed_viewers(&mut ui_b, &before_b).await; + eprintln!( + "HANDOVER_EVENTS_DELIVERED replacement_manifest_requests={} replacement_segment_requests={}", + fixture.b.manifests.load(Ordering::Acquire), + fixture.b.segments.load(Ordering::Acquire) + ); + let after = snapshot(&mut ui_a).await; + assert_eq!(after.len(), 2); + assert!(after.iter().all(|row| row.provider.as_ref() == "account-b")); + if manual { + eprintln!("BROWSER_SWITCHED web={} control={}", fixture.base, fixture.control); + fixture.proceed.notified().await; + } + fixture.process.kill().await.unwrap(); +} diff --git a/backend/core/src/utils/network/content_coding.rs b/backend/core/src/utils/network/content_coding.rs index aca047ca9..3826a878f 100644 --- a/backend/core/src/utils/network/content_coding.rs +++ b/backend/core/src/utils/network/content_coding.rs @@ -793,7 +793,7 @@ mod tests { let response = local_response(StatusCode::OK, &[], encoded_binary.clone()).await; let mut untouched = decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("identity setup"); - assert!(untouched.decoded_from.is_empty()); + assert_eq!(untouched.decoded_from.len(), 0); assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded_binary); } @@ -806,7 +806,7 @@ mod tests { .await .expect("identity setup"); - assert!(decoded.decoded_from.is_empty()); + assert_eq!(decoded.decoded_from.len(), 0); assert_eq!(decoded.content_coding_observation(), None); assert!(!decoded.headers.contains_key(CONTENT_ENCODING)); assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("identity body"), encoded); diff --git a/backend/core/src/utils/network/request.rs b/backend/core/src/utils/network/request.rs index d914568a6..e189356fc 100644 --- a/backend/core/src/utils/network/request.rs +++ b/backend/core/src/utils/network/request.rs @@ -52,11 +52,11 @@ pub enum ResourceDestination { } /// How long a verdict that allows a URL to be handed to a client is remembered. -const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_secs(60); +const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_mins(1); /// How long a verdict that keeps a destination away from clients is remembered. Both classes fail /// closed, so they can be cached longer than the one that decides exposure. -const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_secs(300); +const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_mins(5); /// How long a name that produced no answer within the resolution budget is remembered. /// diff --git a/backend/core/src/utils/network/request/tests/behavior.rs b/backend/core/src/utils/network/request/tests/behavior.rs index 2a529adf3..96516cd2b 100644 --- a/backend/core/src/utils/network/request/tests/behavior.rs +++ b/backend/core/src/utils/network/request/tests/behavior.rs @@ -108,7 +108,7 @@ async fn atomic_download_temp_files_are_unique_and_use_target_directory() { drop(second_output); drop(first_temp); drop(second_temp); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[test] @@ -181,7 +181,7 @@ async fn local_epg_file_respects_max_download_bytes() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -213,7 +213,7 @@ async fn file_url_epg_source_respects_streamed_size_limit() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -243,7 +243,7 @@ async fn local_epg_replace_error_cleans_temp_file() { .expect_err("replacing a directory should fail"); assert!(persist.is_dir()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -273,7 +273,7 @@ async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() { .expect_err("reading a directory as an EPG file should fail"); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[tokio::test] @@ -301,7 +301,7 @@ async fn unsupported_epg_url_scheme_is_rejected_without_network_access() { assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'")); assert!(!persist.exists()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } #[cfg(windows)] @@ -370,7 +370,7 @@ async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() { assert!(err.to_string().contains("exceeds configured limit")); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } @@ -409,7 +409,7 @@ async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file() .expect_err("truncated response body should fail"); assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } @@ -466,7 +466,7 @@ async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomic assert_eq!(accepted.load(Ordering::SeqCst), 2); assert_eq!(max_active.load(Ordering::SeqCst), 1); assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); server_handle.abort(); } diff --git a/backend/core/src/utils/network/request/tests/persistence.rs b/backend/core/src/utils/network/request/tests/persistence.rs index e4b1fa9a0..56032c4a1 100644 --- a/backend/core/src/utils/network/request/tests/persistence.rs +++ b/backend/core/src/utils/network/request/tests/persistence.rs @@ -37,5 +37,5 @@ async fn file_url_epg_source_is_copied_to_persist_path() { tokio::fs::read_to_string(&persist).await.expect("persisted content"), "BEGIN:VCALENDAR\nEND:VCALENDAR\n" ); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0); } diff --git a/backend/curation/src/coordinator/tests.rs b/backend/curation/src/coordinator/tests.rs index 0dd49fded..3f8e085ea 100644 --- a/backend/curation/src/coordinator/tests.rs +++ b/backend/curation/src/coordinator/tests.rs @@ -156,7 +156,7 @@ async fn empty_and_no_match_are_complete_but_disabled_sources_are_not_configured let CurationRunOutcome::Complete(result) = run(&config(None), &catalog(), &server).await else { panic!("valid empty or no-match is complete") }; - assert!(result.memberships.is_empty()); + assert_eq!(result.memberships.len(), 0); assert_eq!(result.selectors.len(), 2); } let server = TestServer::new(http_response(200, PAGE)).await; diff --git a/backend/curation/src/kernel.rs b/backend/curation/src/kernel.rs index 674e8eaf4..6c6bc8a27 100644 --- a/backend/curation/src/kernel.rs +++ b/backend/curation/src/kernel.rs @@ -699,7 +699,7 @@ mod tests { panic!("an empty source is complete") }; assert_eq!(reference_count, 0); - assert!(memberships.is_empty()); + assert_eq!(memberships.len(), 0); let no_match = vec![reference(CurationMediaKind::Movie, "Missing", None, Some(8), Some(1))]; let SelectorOutcome::Complete { reference_count, memberships, .. } = @@ -708,7 +708,7 @@ mod tests { panic!("a complete source with no local match is complete") }; assert_eq!(reference_count, 1); - assert!(memberships.is_empty()); + assert_eq!(memberships.len(), 0); let matched = vec![reference(CurationMediaKind::Movie, "Matched", None, Some(7), Some(1))]; let SelectorOutcome::Complete { reference_count, memberships, .. } = diff --git a/backend/curation/src/test_support.rs b/backend/curation/src/test_support.rs index e4d6b7cff..fca33e304 100644 --- a/backend/curation/src/test_support.rs +++ b/backend/curation/src/test_support.rs @@ -36,7 +36,7 @@ impl TestServer { } pub(crate) async fn serve(responses: Vec>, delay: Duration, send_headers: bool) -> Self { - assert!(!responses.is_empty()); + assert_ne!(responses.len(), 0); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}/", listener.local_addr().unwrap()); let requests = Arc::new(Mutex::new(Vec::new())); diff --git a/backend/curation/src/tmdb/client/tests.rs b/backend/curation/src/tmdb/client/tests.rs index db054bf11..6eeae2d0a 100644 --- a/backend/curation/src/tmdb/client/tests.rs +++ b/backend/curation/src/tmdb/client/tests.rs @@ -136,7 +136,7 @@ async fn tmdb_empty_is_valid_only_at_the_start_with_coherent_totals() { for pages in [0, 1] { let body = json!({"page":1,"total_pages":pages,"total_results":0,"results":[]}).to_string(); let server = TestServer::new(http_response(200, &body)).await; - assert!(client(&server).fetch(&selector(100)).await.unwrap().is_empty()); + assert_eq!(client(&server).fetch(&selector(100)).await.unwrap().len(), 0); assert_eq!(server.requests.lock().unwrap().len(), 1); } for body in [ @@ -222,7 +222,7 @@ async fn tmdb_all_routes_preserve_headers_with_no_query_token_or_details() { let mut selector = selector(100); selector.kind = kind; selector.time_window = window; - assert!(client.fetch(&selector).await.unwrap().is_empty()); + assert_eq!(client.fetch(&selector).await.unwrap().len(), 0); let requests = server.requests.lock().unwrap(); assert_eq!(requests.len(), 1); assert!( @@ -285,7 +285,7 @@ async fn tmdb_identity_chunked_and_gzip_exact_and_plus_one_count_decoded_bytes() let mut budget = client.batch_budget(); let result = client.trending(&selector(100), &mut budget).await; if length == BODY_LIMIT { - assert!(result.unwrap().is_empty()); + assert_eq!(result.unwrap().len(), 0); } else { assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit); } diff --git a/backend/curation/src/trakt/mod.rs b/backend/curation/src/trakt/mod.rs index 6d1a10b5a..32cd230dc 100644 --- a/backend/curation/src/trakt/mod.rs +++ b/backend/curation/src/trakt/mod.rs @@ -343,7 +343,7 @@ mod tests { assert_eq!(evaluation.selectors.len(), 1); assert_eq!(evaluation.selectors[0].reference_count, 0); assert_eq!(evaluation.selectors[0].membership_count, 0); - assert!(evaluation.memberships.is_empty()); + assert_eq!(evaluation.memberships.len(), 0); server.abort(); } diff --git a/backend/dvr/src/recording/recording_conflict.rs b/backend/dvr/src/recording/recording_conflict.rs index b26a5d05c..d6832b3f9 100644 --- a/backend/dvr/src/recording/recording_conflict.rs +++ b/backend/dvr/src/recording/recording_conflict.rs @@ -320,7 +320,7 @@ mod tests { None, ); assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict); - assert!(preview.overlap_segments.is_empty()); + assert_eq!(preview.overlap_segments.len(), 0); } #[test] diff --git a/backend/dvr/src/recording/recording_deletion.rs b/backend/dvr/src/recording/recording_deletion.rs index 97af7265e..36079a56e 100644 --- a/backend/dvr/src/recording/recording_deletion.rs +++ b/backend/dvr/src/recording/recording_deletion.rs @@ -706,7 +706,7 @@ mod tests { let dir = TempDir::new().expect("tempdir"); let (queue, shared_file, target) = delete_one_of_two_entries_sharing(&dir).await; assert!(target.still_referenced, "Bob still holds this file"); - assert!(target.paths_to_unlink().is_empty()); + assert_eq!(target.paths_to_unlink().len(), 0); let unlinked = execute_deletion_target(&target).await.expect("execute"); assert_eq!(unlinked, None, "nothing was unlinked"); assert!(shared_file.exists(), "Bob's recording must survive Alice's deletion"); diff --git a/backend/dvr/src/recording/recording_queue/tests/policy.rs b/backend/dvr/src/recording/recording_queue/tests/policy.rs index feb57ac94..1787fad51 100644 --- a/backend/dvr/src/recording/recording_queue/tests/policy.rs +++ b/backend/dvr/src/recording/recording_queue/tests/policy.rs @@ -233,7 +233,7 @@ fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again() }; let promoted = promote_from_queue(&mut candidate); assert!(promoted.is_none(), "nothing needs to run"); - assert!(candidate.active.is_empty()); + assert_eq!(candidate.active.len(), 0); assert!(candidate.queue.is_empty(), "bob left the queue"); let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed"); assert_eq!(bob.state, RecordingTaskState::Completed); diff --git a/backend/dvr/src/recording/recording_retention.rs b/backend/dvr/src/recording/recording_retention.rs index 88ebc22b0..621ea42a3 100644 --- a/backend/dvr/src/recording/recording_retention.rs +++ b/backend/dvr/src/recording/recording_retention.rs @@ -355,7 +355,7 @@ mod tests { let mut t = completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1); t.recording.completed_at = None; let out = compute_candidates(&[t], &config, 1_000_000_000); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -593,6 +593,6 @@ mod tests { let config = RetentionConfig::default(); let tasks = vec![completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1)]; let out = compute_candidates(&tasks, &config, 0); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } } diff --git a/backend/dvr/src/recording/recording_rule_scheduler.rs b/backend/dvr/src/recording/recording_rule_scheduler.rs index 90513581f..fc671c9c6 100644 --- a/backend/dvr/src/recording/recording_rule_scheduler.rs +++ b/backend/dvr/src/recording/recording_rule_scheduler.rs @@ -442,7 +442,7 @@ mod tests { let key = occurrence_key("r1", &source(), "ch-1", 1_000, "t:my show"); let tasks = vec![task("r1", &key)]; let out = plan_materializations(&rules, &programmes, &tasks, &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -450,7 +450,7 @@ mod tests { let rules = vec![new_episode_rule()]; let programmes = vec![epg_programme("My Show", -1, 100)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -458,7 +458,7 @@ mod tests { let rules = vec![new_episode_rule()]; let programmes = vec![epg_programme("My Show", 5_000, 6_000)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(2_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -468,7 +468,7 @@ mod tests { p.previously_shown = true; let programmes = vec![p]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -478,7 +478,7 @@ mod tests { let rules = vec![r]; let programmes = vec![epg_programme("My Show", 1_000, 2_000)]; let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -502,7 +502,7 @@ mod tests { // Set the EPG horizon to 1 hour — the next Sunday is // days away. let out = plan_materializations(&rules, &[], &[], &TombstoneSet::default(), now, Some(3_600)); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } #[test] @@ -514,6 +514,6 @@ mod tests { let out = plan_materializations(&[rule], &[], &[], &TombstoneSet::default(), 0, None); - assert!(out.is_empty()); + assert_eq!(out.len(), 0); } } diff --git a/backend/dvr/src/recording/recording_worker.rs b/backend/dvr/src/recording/recording_worker.rs index 6a5eba473..9e3c909e6 100644 --- a/backend/dvr/src/recording/recording_worker.rs +++ b/backend/dvr/src/recording/recording_worker.rs @@ -405,7 +405,7 @@ pub async fn recovery_decision_for(final_path: &Path, partial: &Path) -> Recover #[cfg(test)] mod tests { - #[cfg(unix)] + #[cfg(target_os = "linux")] use super::spawn_retrying_busy_executable; use super::{ build_recording_args, classify_ffmpeg_failure, recording_partial_path, @@ -778,7 +778,8 @@ mod tests { let _ = fs::remove_dir_all(&recording.file_dir); } - #[cfg(unix)] + // Linux rejects execution while a writer is open; macOS permits it. + #[cfg(target_os = "linux")] #[tokio::test] async fn spawn_retries_while_executable_is_open_for_writing() { let script = fake_ffmpeg_script("busy", "#!/bin/sh\nexit 0\n"); @@ -970,8 +971,7 @@ mod tests { }; assert!(generated.status.success(), "{}", String::from_utf8_lossy(&generated.stderr)); let bytes = tokio::fs::read(&input).await?; - // This address is outside the usual environment's localhost exclusion. - let listener = tokio::net::TcpListener::bind("127.0.0.2:0").await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let url = reqwest::Url::parse(&format!("http://{}/api/v1/playlist/recording/secret/live", listener.local_addr()?))?; let (request_tx, request_rx) = tokio::sync::oneshot::channel(); @@ -991,7 +991,12 @@ mod tests { Ok::<_, std::io::Error>(()) }); let wrapper = directory.path().join("ffmpeg-with-proxy"); - tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?; + // Explicitly remove proxy exclusions so localhost still exercises the bypass. + tokio::fs::write( + &wrapper, + "#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n", + ) + .await?; fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?; let mut recording = make_recording(chrono::Utc::now().timestamp(), 60); recording.url = url; diff --git a/backend/dvr/src/recording/recording_ws.rs b/backend/dvr/src/recording/recording_ws.rs index 8684f9920..fd70f49b0 100644 --- a/backend/dvr/src/recording/recording_ws.rs +++ b/backend/dvr/src/recording/recording_ws.rs @@ -276,7 +276,7 @@ mod tests { let queue = RecordingQueue::new(); let snapshot = recording_snapshot(&queue, &no_subject_claims(), &bare_app_config()).await; assert_eq!(snapshot.revision.0, 0); - assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.tasks.len(), 0); assert_eq!(snapshot.quota, RecordingQuotaSummaryDto::default(), "a session with no subject is charged nothing"); } @@ -285,7 +285,7 @@ mod tests { let queue = RecordingQueue::new(); let snapshot = recording_snapshot(&queue, &no_read_claims(), &bare_app_config()).await; assert_eq!(snapshot.revision.0, 0); - assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.tasks.len(), 0); } /// Every internal fact a recording carries that a client must never diff --git a/backend/hls/src/availability/manifest_failure.rs b/backend/hls/src/availability/manifest_failure.rs new file mode 100644 index 000000000..8c969ea10 --- /dev/null +++ b/backend/hls/src/availability/manifest_failure.rs @@ -0,0 +1,46 @@ +//! Publish degraded origin evidence before the manifest retry loop starts waiting. + +use super::reevaluation::register_terminal_reserve_reevaluation; +use crate::{ + manifest_fetch::{HlsOriginManifestFetchContext, OriginManifestFetchError}, + origin_progress::HlsOriginPathCondition, + post_refresh_availability::commit_post_refresh_terminal_fallback, + refresh::{HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason}, +}; +use std::sync::Arc; + +pub(crate) async fn record_retryable_manifest_attempt_failure( + context: &HlsOriginManifestFetchContext, + expected_progress_generation: u64, + error: &OriginManifestFetchError, + now_ms: u64, +) { + let action = { + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() + || session.origin_control.progress_generation != expected_progress_generation + { + return; + } + if matches!(error, OriginManifestFetchError::RetryableStatus(..)) { + session.origin_control.record_origin_response(now_ms); + } + // An individual soft failure must not weaken an already established hard failure or conflict. + if matches!( + session.origin_control.path_condition, + HlsOriginPathCondition::ProgressExpected | HlsOriginPathCondition::RetryableFetchFailure + ) { + session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; + } + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + origin_progress_generation: session.origin_control.progress_generation, + media_readiness_generation: session.activity.media_readiness_generation, + } + }; + let Some(ctx) = context.availability_runtime.as_ref().and_then(crate::hls_ctx::WeakHlsCtx::upgrade) else { + return; + }; + let registration = register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(&context.session)).await; + let _ = commit_post_refresh_terminal_fallback(ctx, Arc::clone(&context.session), action, registration).await; +} diff --git a/backend/hls/src/availability/mod.rs b/backend/hls/src/availability/mod.rs index fdfdc49f5..8e3ddc166 100644 --- a/backend/hls/src/availability/mod.rs +++ b/backend/hls/src/availability/mod.rs @@ -17,11 +17,18 @@ use super::{ HlsLiveReserveDeadline, HlsOriginProgressDecision, ProxySessionId, }; +mod manifest_failure; mod recovery_pressure; mod reevaluation; mod terminal_cutover; mod terminal_pending_owner; -pub(crate) use self::terminal_cutover::commit_terminal_tail_if_lease_reserve_requires_cutover_detailed; +pub(crate) use self::{ + manifest_failure::record_retryable_manifest_attempt_failure, + terminal_cutover::{ + commit_terminal_tail_if_lease_reserve_requires_cutover_detailed, terminal_bundle_failure_compatibility, + terminal_bundle_incompatibility, + }, +}; use self::{ recovery_pressure::{ evaluate_and_commit_session_recovery_pressure_in_snapshot, hls_recovery_pressure_policy, diff --git a/backend/hls/src/availability/reevaluation.rs b/backend/hls/src/availability/reevaluation.rs index 177ff7e05..b8028a006 100644 --- a/backend/hls/src/availability/reevaluation.rs +++ b/backend/hls/src/availability/reevaluation.rs @@ -347,7 +347,7 @@ async fn evaluate_post_refresh_terminal_attempt( cycle: &mut HlsAvailabilityReevaluationCycle, now_ms: u64, ) -> HlsTerminalReevaluationDecision { - if !matches!(mode, HlsAvailabilityReevaluationMode::PostRefresh(_)) { + if matches!(mode, HlsAvailabilityReevaluationMode::RecoveryPressure) { return HlsTerminalReevaluationDecision::Continue { terminal_evaluation_pending: false }; } match evaluate_active_terminal_leases_for_reevaluation(ctx, session, &owner_key.proxy_session_id, now_ms).await { @@ -439,7 +439,7 @@ async fn run_hls_availability_reevaluation_attempt( session: &mut HlsSessionHandle, owner_key: &mut HlsAvailabilityReevaluationOwnerKey, ownership: &HlsAvailabilityReevaluationOwnership, - refresh_request: &mut OriginRefreshRequest, + refresh_request: &mut Option, cycle: &mut HlsAvailabilityReevaluationCycle, ) -> HlsAvailabilityCycleDecision { if !ownership.is_current(owner_key) { @@ -463,6 +463,9 @@ async fn run_hls_availability_reevaluation_attempt( HlsTerminalReevaluationDecision::RestartCycle => return HlsAvailabilityCycleDecision::RestartCycle, HlsTerminalReevaluationDecision::Stop => return HlsAvailabilityCycleDecision::Stop, }; + let Some(refresh_request) = refresh_request.as_mut() else { + return HlsAvailabilityCycleDecision::FinishCycle; + }; let schedule = match evaluate_availability_refresh_attempt( ctx, session, @@ -482,10 +485,13 @@ async fn run_hls_availability_reevaluation_attempt( return HlsAvailabilityCycleDecision::FinishCycle; } let retry_from_ms = ctx.hls_proxy.now_ms(); - let Some(retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else { + let Some(mut retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else { warn!("HLS availability reevaluation cycle stopped: reason=deadline_elapsed"); return HlsAvailabilityCycleDecision::FinishCycle; }; + if let Some(deadline) = cycle.live_reserve_deadline { + retry_at_ms = retry_at_ms.min(deadline.next_reevaluation_at_ms); + } match wait_for_availability_owner_signal(ownership, retry_at_ms, retry_from_ms).await { HlsAvailabilityOwnerWaitOutcome::Cancelled => { ownership.discard_superseded(owner_key); @@ -504,7 +510,7 @@ async fn run_hls_availability_reevaluation( mut session: HlsSessionHandle, mut owner_key: HlsAvailabilityReevaluationOwnerKey, ownership: HlsAvailabilityReevaluationOwnership, - mut refresh_request: OriginRefreshRequest, + mut refresh_request: Option, ) { 'cycles: loop { let mut cycle = HlsAvailabilityReevaluationCycle::new(ctx.hls_proxy.now_ms()); @@ -605,6 +611,24 @@ pub(super) fn register_hls_availability_reevaluation_with_mode( let coordinator = ctx.hls_proxy.availability_reevaluations(); let registration_key = owner_key.clone(); coordinator.register(registration_key, mode, move |ownership| { - run_hls_availability_reevaluation(ctx, session, owner_key, ownership, refresh_request) + run_hls_availability_reevaluation(ctx, session, owner_key, ownership, Some(refresh_request)) + }) +} + +pub(super) fn register_terminal_reserve_reevaluation( + ctx: HlsCtx, + session: HlsSessionHandle, +) -> futures::future::BoxFuture<'static, HlsAvailabilityReevaluationRegistration> { + Box::pin(async move { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let Some(owner_key) = ctx.hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await + else { + return HlsAvailabilityReevaluationRegistration::Superseded; + }; + ctx.hls_proxy.availability_reevaluations().register( + owner_key.clone(), + HlsAvailabilityReevaluationMode::TerminalReserve, + move |ownership| run_hls_availability_reevaluation(ctx, session, owner_key, ownership, None), + ) }) } diff --git a/backend/hls/src/availability/terminal_cutover.rs b/backend/hls/src/availability/terminal_cutover.rs index b3f5edc7d..f21e9d92c 100644 --- a/backend/hls/src/availability/terminal_cutover.rs +++ b/backend/hls/src/availability/terminal_cutover.rs @@ -8,9 +8,9 @@ use super::{ recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget}, terminal_pending_owner::{ - commit_prepared_terminal_unavailable_after_owner_failure, commit_terminal_unavailable, - register_ready_terminal_pending_owner, register_terminal_pending_owner, terminal_asset_revision_guard, - terminal_resolution_with_failed_closed_fallback, + commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure, + commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner, + terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback, }, HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution, HLS_PLAYBACK_RATE_GUARD_MILLI, @@ -184,6 +184,7 @@ async fn resolve_terminal_cutover_before_commit_window( &context.lease.lease_id, &preparation, context.now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, ), context.evaluation.safe_deadline, ); @@ -239,7 +240,7 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover( lease: &HlsAccessLease, now_ms: u64, ) -> HlsTerminalResolution { - commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( + let detailed = commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( ctx, session, proxy_session_id, @@ -247,8 +248,30 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover( now_ms, HlsTerminalDecisionPurpose::OrdinaryCutover, ) - .await - .resolution + .await; + if detailed.live_reserve_deadline.is_some() + && matches!(detailed.resolution, HlsTerminalResolution::LiveAllowed | HlsTerminalResolution::Reevaluate) + { + let registration = + super::reevaluation::register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(session)).await; + if matches!( + registration, + crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::CapacityExceeded + | crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::RuntimeUnavailable + ) { + return commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( + ctx, + session, + proxy_session_id, + lease, + now_ms, + HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback, + ) + .await + .resolution; + } + } + detailed.resolution } pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detailed( @@ -341,13 +364,26 @@ async fn commit_prepared_terminal_decision( preparation: &HlsTerminalTailPreparation, now_ms: u64, ) -> HlsTerminalResolution { + if !preparation.trigger.is_runtime_policy() + && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + { + return commit_late_custom_response(HlsTerminalCommitContext { + ctx, + session, + proxy_session_id, + lease_id, + preparation, + now_ms, + }) + .await; + } let reason = preparation.trigger.reason(); let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) { Ok(asset) => asset, Err(compatibility) => { let context = HlsTerminalCommitContext { ctx, session, proxy_session_id, lease_id, preparation, now_ms }; let outcome = commit_terminal_unavailable(context, None, compatibility.terminal_compatibility()); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } }; commit_prepared_runtime_custom_tail(ctx, session, proxy_session_id, lease_id, preparation, now_ms, asset).await @@ -366,6 +402,11 @@ pub async fn commit_prepared_runtime_custom_tail( if preparation.trigger.reason() != asset.reason { return HlsTerminalResolution::Reevaluate; } + if !preparation.trigger.is_runtime_policy() + && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + { + return commit_late_custom_response(context).await; + } let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset); let media_asset = asset.asset; let static_incompatibility = if preparation.manifest_snapshot.delivery_mode @@ -381,7 +422,7 @@ pub async fn commit_prepared_runtime_custom_tail( }; if let Some(reason) = static_incompatibility { let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } let bundle_key = prepared_terminal_bundle_key( &media_asset, @@ -403,7 +444,7 @@ pub async fn commit_prepared_runtime_custom_tail( Some(expected_asset), HlsTerminalTailCompatibility::AssetRevisionMismatch, ); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } HlsPreparedTerminalBundleState::Preparing { .. } => { return register_terminal_pending_owner(context, media_asset, expected_asset, bundle_key).await; @@ -411,12 +452,12 @@ pub async fn commit_prepared_runtime_custom_tail( HlsPreparedTerminalBundleState::Failed { key, reason } => { let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } HlsPreparedTerminalBundleState::Incompatible { reason, .. } => { let reason = terminal_bundle_incompatibility(reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason); - return terminal_resolution_with_failed_closed_fallback(context, outcome); + return terminal_resolution_with_failed_closed_fallback(context, outcome).await; } }; if preparation.trigger.is_runtime_policy() { @@ -426,10 +467,11 @@ pub async fn commit_prepared_runtime_custom_tail( expected_asset, bundle_key, prepared_bundle, - ); + ) + .await; } let outcome = commit_ready_terminal_bundle(context, media_asset, expected_asset, bundle_key, prepared_bundle).await; - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } struct HlsReadyTerminalSplice { @@ -624,7 +666,7 @@ pub(super) async fn commit_ready_terminal_bundle( } } -pub(super) fn terminal_bundle_incompatibility( +pub(crate) fn terminal_bundle_incompatibility( reason: HlsPreparedTerminalBundleIncompatibility, ) -> HlsTerminalTailCompatibility { match reason { @@ -636,7 +678,7 @@ pub(super) fn terminal_bundle_incompatibility( } } -pub(super) fn terminal_bundle_failure_compatibility( +pub(crate) fn terminal_bundle_failure_compatibility( actual_key: HlsPreparedTerminalBundleKey, required_key: HlsPreparedTerminalBundleKey, reason: HlsPreparedTerminalBundleFailure, diff --git a/backend/hls/src/availability/terminal_pending_owner.rs b/backend/hls/src/availability/terminal_pending_owner.rs index 7a2f88fb1..00ad7f6fc 100644 --- a/backend/hls/src/availability/terminal_pending_owner.rs +++ b/backend/hls/src/availability/terminal_pending_owner.rs @@ -89,7 +89,7 @@ pub(super) async fn register_terminal_pending_owner( ownership, ) }); - terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration) + terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await } fn terminal_pending_owner_key( @@ -117,7 +117,7 @@ fn terminal_pending_owner_key( }) } -pub(super) fn register_ready_terminal_pending_owner( +pub(super) async fn register_ready_terminal_pending_owner( context: HlsTerminalCommitContext<'_>, asset: Arc, expected_asset: HlsRuntimeCustomTailAssetIdentity, @@ -150,16 +150,15 @@ pub(super) fn register_ready_terminal_pending_owner( }; let outcome = commit_ready_terminal_bundle(task_context, asset, expected_asset, bundle_key, bundle).await; if ownership.is_current() { - observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback( - task_context, - outcome, - )); + observe_autonomous_terminal_resolution( + terminal_resolution_with_failed_closed_fallback(task_context, outcome).await, + ); } }); - terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration) + terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await } -pub(super) fn terminal_resolution_for_pending_registration( +pub(super) async fn terminal_resolution_for_pending_registration( context: HlsTerminalCommitContext<'_>, expected_asset: HlsRuntimeCustomTailAssetIdentity, latest_safe_commit_at_ms: u64, @@ -178,7 +177,7 @@ pub(super) fn terminal_resolution_for_pending_registration( Some(expected_asset), HlsTerminalTailCompatibility::TerminalMediaNotReady, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } } } @@ -193,7 +192,7 @@ async fn terminal_resolution_after_settled_bundle_observation( match state { Some(HlsPreparedTerminalBundleState::Ready { bundle }) if bundle.matches_key_and_shape(bundle_key) => { let outcome = commit_ready_terminal_bundle(context, asset, expected_asset, bundle_key, bundle).await; - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Ready { .. }) => { let outcome = commit_terminal_unavailable( @@ -201,12 +200,12 @@ async fn terminal_resolution_after_settled_bundle_observation( Some(expected_asset), HlsTerminalTailCompatibility::AssetRevisionMismatch, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Failed { key, reason }) => { let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason); let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Incompatible { key, reason }) => { let compatibility = if key == bundle_key { @@ -215,7 +214,7 @@ async fn terminal_resolution_after_settled_bundle_observation( HlsTerminalTailCompatibility::AssetRevisionMismatch }; let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } Some(HlsPreparedTerminalBundleState::Preparing { .. }) | None => { let outcome = commit_terminal_unavailable( @@ -223,7 +222,7 @@ async fn terminal_resolution_after_settled_bundle_observation( Some(expected_asset), HlsTerminalTailCompatibility::TerminalMediaNotReady, ); - terminal_resolution_with_failed_closed_fallback(context, outcome) + terminal_resolution_with_failed_closed_fallback(context, outcome).await } } } @@ -330,7 +329,7 @@ pub(super) async fn run_terminal_pending_owner( commit_terminal_unavailable(context, Some(expected_asset), compatibility) } }; - observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome)); + observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome).await); } fn terminal_pending_retry_after_ms(now_ms: u64, latest_safe_commit_at_ms: u64) -> u64 { @@ -378,7 +377,7 @@ pub(super) fn terminal_resolution_for_commit_outcome( } } -pub(super) fn terminal_resolution_with_failed_closed_fallback( +pub(super) async fn terminal_resolution_with_failed_closed_fallback( context: HlsTerminalCommitContext<'_>, outcome: HlsTerminalCommitOutcome, ) -> HlsTerminalResolution { @@ -389,6 +388,9 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback( if context.preparation.trigger.is_runtime_policy() { return resolution; } + if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed { + return commit_late_custom_response(context).await; + } commit_prepared_terminal_unavailable_after_owner_failure( context.ctx, context.session, @@ -396,9 +398,48 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback( context.lease_id, context.preparation, context.now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, ) } +pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution { + let prepared = crate::late_custom_response::prepare_late_custom_response( + context.ctx, + context.proxy_session_id, + context.lease_id, + context.preparation, + context.now_ms, + ) + .await; + let (plan, expected_asset) = match prepared { + Ok(prepared) => prepared, + Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => { + return HlsTerminalResolution::Reevaluate + } + Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => { + return commit_prepared_terminal_unavailable_after_owner_failure( + context.ctx, + context.session, + context.proxy_session_id, + context.lease_id, + context.preparation, + context.now_ms, + reason, + ) + } + }; + let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: context.session, + lease_id: context.lease_id, + proxy_session_id: context.proxy_session_id, + preparation: context.preparation, + now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms), + payload: HlsTerminalCommitPayload::CustomResponse(plan), + asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)), + }); + terminal_resolution_for_commit_outcome(outcome, context.now_ms) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum HlsAutonomousTerminalObservation { Committed, @@ -486,6 +527,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure( lease_id: &HlsAccessLeaseId, preparation: &HlsTerminalTailPreparation, now_ms: u64, + reason: HlsTerminalTailCompatibility, ) -> HlsTerminalResolution { let expected_asset = configured_terminal_asset_identity(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable); let outcome = ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { @@ -494,9 +536,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure( proxy_session_id, preparation, now_ms: ctx.hls_proxy.terminal_commit_now_ms().max(now_ms), - payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure( - HlsTerminalTailCompatibility::TerminalMediaNotReady, - ), + payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(reason), asset_revision_guard: terminal_asset_revision_guard( ctx, HlsRuntimeCustomTailReason::ChannelUnavailable, diff --git a/backend/hls/src/availability/tests.rs b/backend/hls/src/availability/tests.rs index bf7f3bd80..3b81087c2 100644 --- a/backend/hls/src/availability/tests.rs +++ b/backend/hls/src/availability/tests.rs @@ -212,6 +212,7 @@ mod policy; mod publication; +mod late_custom_response; mod recovery; mod storage; diff --git a/backend/hls/src/availability/tests/late_custom_response.rs b/backend/hls/src/availability/tests/late_custom_response.rs new file mode 100644 index 000000000..fc3f001cd --- /dev/null +++ b/backend/hls/src/availability/tests/late_custom_response.rs @@ -0,0 +1,532 @@ +use super::*; +use crate::{ + late_custom_response::prepare_late_custom_response, + manager::{HlsTerminalCommitPayload, HlsTerminalCommitRequest}, +}; + +async fn late_fixture(name: &str) -> HlsTerminalPendingCommitFixture { + let fixture = terminal_pending_commit_fixture(name).await; + fixture.session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + fixture +} + +async fn prepare( + fixture: &HlsTerminalPendingCommitFixture, +) -> (Arc, HlsRuntimeCustomTailAssetIdentity) { + prepare_late_custom_response( + &fixture.ctx, + &fixture.proxy_session_id, + &fixture.lease_id, + &fixture.preparation, + fixture.now_ms, + ) + .await + .unwrap_or_else(|_| panic!("late custom plan")) +} + +fn commit( + fixture: &HlsTerminalPendingCommitFixture, + plan: Arc, + expected_asset: HlsRuntimeCustomTailAssetIdentity, + now_ms: u64, +) -> HlsTerminalCommitOutcome { + fixture.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &fixture.session, + lease_id: &fixture.lease_id, + proxy_session_id: &fixture.proxy_session_id, + preparation: &fixture.preparation, + now_ms, + payload: HlsTerminalCommitPayload::CustomResponse(plan), + asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, expected_asset.reason, Some(expected_asset)), + }) +} + +#[tokio::test] +async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() { + let fixture = late_fixture("late-without-base").await; + let (plan, asset) = prepare(&fixture).await; + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media); + assert!( + fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms) + .await + .is_err(), + "preparation cannot authorize uncommitted custom media" + ); + fixture.session.write().await.segments.clear(); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n")); + assert!(!plan.manifest_body().contains("/hls/shared/live/")); + let segment = fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms) + .await + .expect("committed late media"); + assert!(!segment.bytes.is_empty()); + fixture.ctx.app_config.custom_stream_response.store(None); + assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted); + let replay = fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1) + .await + .expect("frozen media after config reload"); + assert_eq!(replay.bytes, segment.bytes); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms) + .await + .is_err()); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms) + .await + .is_err()); + assert!(fixture + .ctx + .hls_proxy + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000) + .await + .is_err()); +} + +#[tokio::test] +async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() { + for changed_asset in [false, true] { + let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await; + let (plan, asset) = prepare(&fixture).await; + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let expected = if changed_asset { + fixture.ctx.app_config.custom_stream_response.store(None); + HlsTerminalCommitOutcome::SupersededGeneration + } else { + fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000); + HlsTerminalCommitOutcome::RecoveryCommitted + }; + assert_eq!(commit(&fixture, plan, asset, now_ms), expected); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + } +} + +#[tokio::test] +async fn late_custom_response_preserves_the_prepared_asset_reason_guard() { + let fixture = late_fixture("late-wrong-asset-reason").await; + let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted) + .expect("configured alternate custom asset"); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!( + commit_prepared_runtime_custom_tail( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + &fixture.preparation, + now_ms, + asset, + ) + .await, + HlsTerminalResolution::Reevaluate + ); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("live lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test] +async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() { + let mut fixture = late_fixture("late-target-duration-error").await; + let mut manifest = fixture.preparation.manifest_snapshot.clone(); + manifest.target_duration_ms = 1_000; + manifest.playlist_duration_ms = 1_000; + manifest.last_visible_media_end_ms = 1_000; + Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000; + let guard = fixture + .ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("publication guard"); + assert!(fixture + .ctx + .hls_proxy + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + manifest, + fixture.now_ms, + ) + .await + .is_committed()); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("lease with short target duration"); + fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot"); + fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; + let expected = HlsTerminalTailCompatibility::TargetDurationExceeded { + asset_ms: fixture.asset.duration_ms(), + target_ms: 1_000, + }; + assert!(matches!(prepare_late_custom_response( + &fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms, + ).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected)); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert_eq!( + super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext { + ctx: &fixture.ctx, + session: &fixture.session, + proxy_session_id: &fixture.proxy_session_id, + lease_id: &fixture.lease_id, + preparation: &fixture.preparation, + now_ms, + }) + .await, + HlsTerminalResolution::Committed + ); + let terminal = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("terminal unavailable lease"); + assert_eq!( + terminal.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { + decision_generation: fixture.preparation.decision_generation, + reason: expected, + } + ); +} + +async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture { + let fixture = late_fixture(name).await; + let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone(); + config.reverse_proxy = + Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto { + hls_cache: Some(shared::model::HlsCacheConfigDto { + session_idle_timeout: shared::model::Secs::new(60), + ..Default::default() + }), + ..Default::default() + })); + fixture.ctx.app_config.config.store(Arc::new(config)); + fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await; + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + assert!(fixture + .ctx + .hls_proxy + .activate_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let (plan, asset) = prepare(&fixture).await; + assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed); + fixture +} + +#[tokio::test] +async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() { + use crate::{ + lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, + HlsMediaActivityCommitOutcome, + }; + let fixture = committed_late_activity_fixture("late-active-media").await; + let manager = &fixture.ctx.hls_proxy; + let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let before = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms) + .await + .expect("terminal lease"); + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); + for index in 1_u16..=11 { + let now_ms = started_at_ms + u64::from(index) * 10_000; + let access = manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms) + .await + .expect("custom media remains available"); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await, + HlsMediaActivityCommitOutcome::Committed + ); + if index == 6 { + for key in [ + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() }, + ] { + manager + .handle_lifecycle_event( + &fixture.ctx.active_users, + &fixture.ctx.active_provider, + HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 }, + now_ms, + ) + .await; + } + } + let lease = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("active terminal lease"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert_eq!(lease.last_seen_at_ms, now_ms); + assert_eq!(lease.valid_until_ms, now_ms + 60_000); + assert!(lease.active_until_ms.is_some_and(|until| until > now_ms)); + assert_eq!(lease.playback_mode, before.playback_mode); + assert_eq!(lease.playback_cursor, before.playback_cursor); + assert_eq!(lease.admission_generation, before.admission_generation); + assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms); + } + let expired_at_ms = started_at_ms + 170_000; + assert!(manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms) + .await + .is_err()); + let expired = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms) + .await + .expect("expired lease"); + assert_eq!(expired.state, HlsAccessLeaseState::Expired); + assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000); +} + +#[tokio::test] +async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() { + use crate::{ + lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, + HlsMediaActivityCommitOutcome, + }; + let fixture = committed_late_activity_fixture("late-idle-media").await; + let manager = &fixture.ctx.hls_proxy; + let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + manager + .handle_lifecycle_event( + &fixture.ctx.active_users, + &fixture.ctx.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseActive { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + due_at_ms: started_at_ms + 30_000, + }, + started_at_ms + 40_000, + ) + .await; + let idle = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("idle lease"); + assert_eq!(idle.state, HlsAccessLeaseState::Idle); + let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); + let mut access = manager + .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000) + .await + .expect("idle custom media"); + let original_generation = access.terminal_generation; + access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1)); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + let unchanged = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("unchanged idle lease"); + assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms); + assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms); + access.terminal_generation = original_generation; + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, + HlsMediaActivityCommitOutcome::Committed + ); + let active = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) + .await + .expect("reactivated lease"); + assert_eq!(active.state, HlsAccessLeaseState::Activated); + assert_eq!(active.valid_until_ms, started_at_ms + 100_000); + assert_eq!( + manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + let expired = manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000) + .await + .expect("expired lease"); + assert_eq!(expired.state, HlsAccessLeaseState::Expired); + assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms); + assert_eq!(expired.valid_until_ms, active.valid_until_ms); +} + +#[tokio::test] +async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() { + let mut fixture = late_fixture("late-contended-commit").await; + let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms(); + fixture.preparation.cutover_timing = + HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None); + let (plan, asset) = prepare(&fixture).await; + let guard = fixture.session.read().await; + assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); + assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); + drop(guard); + let committed = tokio::time::timeout(Duration::from_secs(3), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + fixture.ctx.hls_proxy.terminal_commit_now_ms(), + ) + .await + .expect("lease"); + if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode { + break committed; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("autonomous late custom commit"); + assert_eq!(committed, plan); + assert_eq!( + commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()), + HlsTerminalCommitOutcome::AlreadyCommitted + ); +} + +#[tokio::test] +async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() { + let mut fixture = late_fixture("late-highwater").await; + for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] { + let guard = fixture + .ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("publication guard"); + let mut manifest = fixture.preparation.manifest_snapshot.clone(); + manifest.first_proxy_seq = last_seq; + manifest.last_proxy_seq = last_seq; + manifest.discontinuity_sequence = discontinuity_sequence; + let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment"); + segment.proxy_seq = last_seq; + segment.discontinuity_before = discontinuity_before; + assert!(fixture + .ctx + .hls_proxy + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + manifest, + fixture.now_ms + ) + .await + .is_committed()); + } + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("lease"); + fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot"); + fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; + let (plan, _) = prepare(&fixture).await; + assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n")); + assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n")); +} + +#[tokio::test] +async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() { + use crate::terminal_commit::{ + terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey, + HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision, + }; + let fixture = late_fixture("late-bounded-cas").await; + let (plan, asset) = prepare(&fixture).await; + let coordinator = HlsTerminalCommitRetryCoordinator::default(); + let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let command = |decision| { + let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding"); + HlsTerminalCommitCommand { + key: HlsTerminalCommitOwnerKey { + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + lease_issued_at_ms: fixture.preparation.lease_issued_at_ms, + decision_generation: fixture.preparation.decision_generation, + }, + session: Arc::clone(&fixture.session), + session_incarnation: fixture + .ctx + .hls_proxy + .sessions() + .session_incarnation(&fixture.session) + .expect("session incarnation"), + preparation: fixture.preparation.clone(), + decision, + media_guard: None, + asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)), + cancellation_epoch, + submission_token, + } + }; + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), + now_ms + ), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); + assert!(matches!( + coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms), + HlsTerminalCommitSubmissionDecision::Attempt { .. } + )); + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), + now_ms + 1 + ), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert!(matches!( + coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert!(matches!( + coordinator.submit( + command(HlsTerminalLeaseDecision::CustomResponse(plan)), + now_ms + terminal_commit_retry_schedule_budget_ms() + ), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); +} diff --git a/backend/hls/src/availability/tests/owner_fixtures.rs b/backend/hls/src/availability/tests/owner_fixtures.rs index 3ae6a2e53..9a5860cd5 100644 --- a/backend/hls/src/availability/tests/owner_fixtures.rs +++ b/backend/hls/src/availability/tests/owner_fixtures.rs @@ -31,6 +31,7 @@ pub(super) fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) - manifest_commit_requirement: super::super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, now_ms: fixture.now_ms, diff --git a/backend/hls/src/availability/tests/recovery.rs b/backend/hls/src/availability/tests/recovery.rs index 218a42c31..cebe93d8f 100644 --- a/backend/hls/src/availability/tests/recovery.rs +++ b/backend/hls/src/availability/tests/recovery.rs @@ -386,3 +386,287 @@ fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_e assert!(pressure.decision.close_admission); assert!(!pressure.decision.evaluate_lease_cutovers); } + +async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeasePlaybackMode { + tokio::time::timeout(Duration::from_millis(4_500), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("active lease"); + if lease.playback_mode != HlsLeasePlaybackMode::Live { + return lease.playback_mode; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("terminal decision before reserve is exhausted") +} + +#[tokio::test] +async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh_is_in_flight() { + let fixture = post_refresh_terminal_fixture_with_progress("autonomous-reserve-deadline", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + { + let mut session = fixture.session.write().await; + session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; + session.origin_refresh.mark_started(fixture.now_ms); + } + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + let result = commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await; + assert_eq!(result, HlsTerminalResolution::LiveAllowed); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + + // R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late. + // Only read the resulting state; do not issue another terminal/manifest request. + let mode = wait_for_terminal_mode(&fixture).await; + assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); + assert!(current_time_millis() < fixture.now_ms + 5_100); + assert!(fixture.session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let fixture = post_refresh_terminal_fixture_with_progress("first-http-failure-cutover", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms, 12_000); + let hits = Arc::new(AtomicUsize::new(0)); + let handler_hits = Arc::clone(&hits); + let router = axum::Router::new().route( + "/index.m3u8", + axum::routing::get(move || { + let handler_hits = Arc::clone(&handler_hits); + async move { + let status = if handler_hits.fetch_add(1, Ordering::SeqCst) == 0 { + axum::http::StatusCode::BAD_GATEWAY + } else { + axum::http::StatusCode::NOT_FOUND + }; + (status, [(axum::http::header::RETRY_AFTER, "60")], "origin failure") + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("local origin binds"); + let address = listener.local_addr().expect("local origin address"); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.expect("local origin serves"); + }); + let mut request = post_refresh_owner_request(&fixture); + request.origin_entry = crate::manifest_fetch::LiveHlsOriginEntry::parse(&format!("http://{address}/index.m3u8")) + .expect("local origin URL"); + let refresh = tokio::spawn(crate::refresh::trigger_origin_refresh_sync(request)); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if fixture.session.read().await.origin_control.path_condition + == HlsOriginPathCondition::RetryableFetchFailure + && fixture.ctx.hls_proxy.availability_reevaluations().owner_count() == 1 + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("first failed attempt publishes degraded evidence and owns the deadline"); + assert_eq!(hits.load(Ordering::SeqCst), 1); + assert!(fixture.session.read().await.origin_refresh.in_flight); + + let mode = wait_for_terminal_mode(&fixture).await; + assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); + assert!(current_time_millis() < fixture.now_ms + 5_100); + assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry"); + assert!(fixture.session.read().await.origin_refresh.in_flight); + assert!(tokio::time::timeout(Duration::from_secs(8), refresh) + .await + .expect("refresh finishes") + .expect("refresh task")); + server.abort(); +} + +#[tokio::test] +async fn recovered_origin_cancels_the_retained_reserve_cutover() { + let fixture = post_refresh_terminal_fixture_with_progress("recovered-reserve-owner", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 18_000, + fixture.now_ms, + ) + .await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::LiveAllowed + ); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + fixture.session.write().await.origin_control.record_media_progress(current_time_millis(), 12_000); + fixture.ctx.hls_proxy.availability_reevaluations().notify_session_evidence_changed(&fixture.proxy_session_id); + wait_for_availability_owner_completion(&fixture).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("recovered lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test] +async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() { + let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 26_000, + fixture.now_ms, + ) + .await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::Committed + ); + let terminal = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("terminal lease"); + assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &terminal, + current_time_millis(), + ) + .await, + HlsTerminalResolution::Committed + ); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test(start_paused = true)] +async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() { + let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await; + fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::LiveAllowed + ); + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; +} + +#[tokio::test] +async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() { + let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 26_000, + fixture.now_ms, + ) + .await; + fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms - 18_000, 12_000); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease"); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms, + ) + .await, + HlsTerminalResolution::Committed + ); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("terminal lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); +} diff --git a/backend/hls/src/availability/tests/runtime_fixtures.rs b/backend/hls/src/availability/tests/runtime_fixtures.rs index 4305fb8e9..a85538356 100644 --- a/backend/hls/src/availability/tests/runtime_fixtures.rs +++ b/backend/hls/src/availability/tests/runtime_fixtures.rs @@ -63,6 +63,7 @@ pub(super) async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFi let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { HlsLeasePlaybackMode::Live => "live", HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", + HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response", HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", HlsLeasePlaybackMode::Ended => "ended", }); @@ -332,7 +333,7 @@ pub(super) async fn assert_multi_lease_fallback_handles_pending_and_unavailable( assert_eq!(aggregate.total, 2); assert_eq!(aggregate.pending_owned, 1); assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}"); - assert!(aggregate.unresolved.is_empty()); + assert_eq!(aggregate.unresolved.len(), 0); assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); let unavailable = fixture .ctx diff --git a/backend/hls/src/availability/tests/terminal_fixtures.rs b/backend/hls/src/availability/tests/terminal_fixtures.rs index 6d0167694..6b5fc6dca 100644 --- a/backend/hls/src/availability/tests/terminal_fixtures.rs +++ b/backend/hls/src/availability/tests/terminal_fixtures.rs @@ -148,6 +148,7 @@ pub(super) async fn install_terminal_pending_ready_base( host_local_index: 0, }, proxy_seq: 40, + media_start_ms: 0, duration_ms: asset.duration_ms(), proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -316,7 +317,8 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail fixture.expected_asset, fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), failure, - ); + ) + .await; assert_eq!(resolution, HlsTerminalResolution::Committed); let lease = fixture @@ -340,6 +342,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l let mut session = fixture.session.write().await; session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; let origin_epoch = session.origin_control.origin_epoch; + let base_end_ms = session.segments[&40].media_start_ms + session.segments[&40].duration_ms; for proxy_seq in 41_u64..=44 { session.segments.insert( proxy_seq, @@ -351,6 +354,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: base_end_ms + (proxy_seq - 41) * 20_000, duration_ms: 20_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/availability_reevaluation.rs b/backend/hls/src/availability_reevaluation.rs index 94e4d8d2c..721c947df 100644 --- a/backend/hls/src/availability_reevaluation.rs +++ b/backend/hls/src/availability_reevaluation.rs @@ -81,6 +81,8 @@ pub enum HlsAvailabilityReevaluationRegistration { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum HlsAvailabilityReevaluationMode { RecoveryPressure, + /// Tracks a live reserve deadline independently of a client request or a running origin fetch. + TerminalReserve, PostRefresh(HlsPostRefreshAvailabilityReason), } @@ -90,8 +92,10 @@ impl HlsAvailabilityReevaluationMode { (Self::PostRefresh(current), Self::PostRefresh(incoming)) => { Self::PostRefresh(stronger_post_refresh_reason(current, incoming)) } - (Self::PostRefresh(reason), Self::RecoveryPressure) - | (Self::RecoveryPressure, Self::PostRefresh(reason)) => Self::PostRefresh(reason), + (Self::PostRefresh(reason), Self::RecoveryPressure | Self::TerminalReserve) + | (Self::RecoveryPressure | Self::TerminalReserve, Self::PostRefresh(reason)) => Self::PostRefresh(reason), + (Self::TerminalReserve, Self::RecoveryPressure | Self::TerminalReserve) + | (Self::RecoveryPressure, Self::TerminalReserve) => Self::TerminalReserve, (Self::RecoveryPressure, Self::RecoveryPressure) => Self::RecoveryPressure, } } @@ -106,10 +110,14 @@ const fn stronger_post_refresh_reason( | (_, HlsPostRefreshAvailabilityReason::HardManifestFailure) => { HlsPostRefreshAvailabilityReason::HardManifestFailure } + (HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, _) + | (_, HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict) => { + HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict + } ( - HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - ) => HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + HlsPostRefreshAvailabilityReason::RetryableManifestFailure, + ) => HlsPostRefreshAvailabilityReason::RetryableManifestFailure, } } @@ -174,7 +182,7 @@ impl HlsAvailabilityReevaluationCoordinator { pub fn register( self: &Arc, key: HlsAvailabilityReevaluationOwnerKey, - mode: HlsAvailabilityReevaluationMode, + mut mode: HlsAvailabilityReevaluationMode, work: Work, ) -> HlsAvailabilityReevaluationRegistration where @@ -186,24 +194,25 @@ impl HlsAvailabilityReevaluationCoordinator { }; let mut state = self.lock_state(); if let Some(current) = state.owners.get_mut(&key.proxy_session_id) { - if current.key == key { - current.mode = current.mode.merge(mode); - current.rerun_requested = true; - current.wake.notify_one(); - return HlsAvailabilityReevaluationRegistration::AlreadyOwned; - } - if !key.supersedes(¤t.key) { + if current.key != key && !key.supersedes(¤t.key) { return HlsAvailabilityReevaluationRegistration::Superseded; } + let installs_refresh_payload = current.mode == HlsAvailabilityReevaluationMode::TerminalReserve + && mode != HlsAvailabilityReevaluationMode::TerminalReserve; + mode = current.mode.merge(mode); // A newer demand for the same proxy session is owned by the // already-counted task. The worker will snapshot current session // evidence and atomically rekey itself without acquiring another // permit. Keeping one dirty bit preserves a bounded successor // cycle while coalescing repeated registrations. - current.mode = current.mode.merge(mode); - current.rerun_requested = true; - current.wake.notify_one(); - return HlsAvailabilityReevaluationRegistration::AlreadyOwned; + // A reserve-only worker has no origin request payload. Replace it + // when recovery work arrives so merging the mode cannot discard that work. + if !installs_refresh_payload { + current.mode = mode; + current.rerun_requested = true; + current.wake.notify_one(); + return HlsAvailabilityReevaluationRegistration::AlreadyOwned; + } } let Ok(task_permit) = Arc::clone(&self.task_capacity).try_acquire_owned() else { return HlsAvailabilityReevaluationRegistration::CapacityExceeded; diff --git a/backend/hls/src/cache.rs b/backend/hls/src/cache.rs index 48099555f..b4497d47e 100644 --- a/backend/hls/src/cache.rs +++ b/backend/hls/src/cache.rs @@ -48,6 +48,8 @@ pub struct StagedCacheObject { pub size: u64, cache_path_generation: Arc, _registration: Arc, + pub(crate) write_budget: Option>, + cleanup: Arc, } /// Typed source used to distinguish a decoded-object size violation from a filesystem failure. @@ -106,6 +108,10 @@ pub struct HlsSegmentCache { capacity_reclaimer: StdRwLock>>, capacity_admission_gate: AsyncMutex<()>, owned_operation_permits: Arc, + #[cfg(test)] + startup_test_checkpoint: StdMutex>, + #[cfg(test)] + delete_test_failures: StdMutex>, } #[derive(Clone)] @@ -125,8 +131,13 @@ struct CacheCapacityState { revision: Arc, active_mutations: HashSet, revision_reservations: HashMap, - /// Revision file bytes found by the last completed usage scan. + next_write_id: u64, + write_owners: HashMap, + abandoned_writes: HashMap, + reserved_spools: HashMap, + /// Current revision bytes, reconciled by scans and successful file mutations. revision_bytes: u64, + revision_metrics: Option>, } #[derive(Default)] @@ -135,17 +146,24 @@ struct CapacityRevision; #[derive(Default)] struct TempFileState { active_files: HashSet, + revision: Arc, deletion_reservations: HashSet, } struct ActiveTempFileRegistration { path: PathBuf, state: Arc>, + changed: Arc, } impl Drop for ActiveTempFileRegistration { fn drop(&mut self) { - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.remove(&self.path); + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.active_files.remove(&self.path) { + state.revision = Arc::new(CapacityRevision); + drop(state); + self.changed.notify_waiters(); + } } } @@ -171,6 +189,8 @@ struct CapacityMutationReservation { session_component: String, replacement: Option<(u64, u64)>, filesystem_mutation_started: bool, + write_budget: Option>, + transferred_bytes: u64, capacity: Arc>, changed: Arc, } @@ -198,6 +218,8 @@ pub struct HlsRevisionDiskReservation { changed: Arc, /// Set when bytes were written under this reservation outside the staged commit accounting. unaccounted_writes: bool, + generation: Arc, + cache_path: PathBuf, } pub struct HlsRevisionFilePin { @@ -217,6 +239,9 @@ mod lifecycle; mod revisions; mod staging; #[cfg(test)] +mod test_support; +mod write_budget; +#[cfg(test)] use capacity::CapacityReservationError; #[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] pub use capacity::{ @@ -226,3 +251,6 @@ pub use capacity::{ pub use error::{hls_cache_capacity_from_io, hls_cache_object_limit_from_io}; pub use keys::HlsCacheObjectKey; pub use staging::CachedSegmentMetadata; +#[cfg(test)] +pub(crate) use test_support::StartupWritePhase; +use write_budget::StagedFileCleanup; diff --git a/backend/hls/src/cache/capacity.rs b/backend/hls/src/cache/capacity.rs index dbb503968..cbd3fca08 100644 --- a/backend/hls/src/cache/capacity.rs +++ b/backend/hls/src/cache/capacity.rs @@ -1,7 +1,7 @@ use super::{ error::{cache_object_limit_error, capacity_error}, hls_cache_capacity_from_io, - lifecycle::{is_temp_cache_file, REWRITE_SECRET_FINGERPRINT_FILE}, + lifecycle::REWRITE_SECRET_FINGERPRINT_FILE, revisions::{is_revision_cache_path, revision_reserved_bytes}, safe_proxy_session_id, CacheCapacityPressure, CacheCapacityState, CapacityInvalidationGuard, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsSegmentCache, @@ -82,11 +82,21 @@ impl CapacityMutationReservation { return Err(CapacityReservationError::Invalidated); } let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size); - let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size); + // Old bytes remain on disk until rename succeeds. Reserve growth now, + // but publish any shrink only after the filesystem mutation completes. + let growth = new_size.saturating_sub(old_size); + let projected_total = capacity.total_bytes.saturating_add(growth); + let projected_session = session_size.saturating_add(growth); let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &self.session_component); - if projected_total.saturating_add(reserved_global) > max_cache_bytes - || projected_session.saturating_add(reserved_session) > max_session_bytes + let transferred = self + .write_budget + .as_ref() + .and_then(|budget| { + capacity.revision_reservations.get(&budget.id).filter(|(owner, _)| owner == &self.session_component) + }) + .map_or(0, |(_, bytes)| (*bytes).min(new_size.saturating_sub(old_size))); + if projected_total.saturating_add(reserved_global.saturating_sub(transferred)) > max_cache_bytes + || projected_session.saturating_add(reserved_session.saturating_sub(transferred)) > max_session_bytes { return Err(CapacityReservationError::Exceeded { pressure: CacheCapacityPressure { @@ -96,10 +106,10 @@ impl CapacityMutationReservation { current_global_bytes: capacity.total_bytes, staged_bytes: new_size, required_session_bytes: projected_session - .saturating_add(reserved_session) + .saturating_add(reserved_session.saturating_sub(transferred)) .saturating_sub(max_session_bytes), required_global_bytes: projected_total - .saturating_add(reserved_global) + .saturating_add(reserved_global.saturating_sub(transferred)) .saturating_sub(max_cache_bytes), }, // Capture the opaque token in the same critical section as the @@ -108,6 +118,12 @@ impl CapacityMutationReservation { revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), }); } + if let Some(budget) = &self.write_budget { + if let Some((_, bytes)) = capacity.revision_reservations.get_mut(&budget.id) { + *bytes -= transferred; + } + } + self.transferred_bytes = transferred; capacity.total_bytes = projected_total; store_session_bytes(&mut capacity, &self.session_component, projected_session); self.replacement = Some((old_size, new_size)); @@ -117,12 +133,36 @@ impl CapacityMutationReservation { pub(super) fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; } pub(super) fn finish_replacement(mut self) { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((old, new)) = self.replacement { + let released = old.saturating_sub(new); + capacity.total_bytes = capacity.total_bytes.saturating_sub(released); + let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); + store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(released)); + if super::revisions::is_revision_cache_path(&self.path) { + capacity.revision_bytes = capacity.revision_bytes.saturating_sub(old).saturating_add(new); + } + } + capacity.abandoned_writes.remove(&self.path); + if let Some(budget) = &self.write_budget { + capacity.write_owners.insert(self.path.clone(), budget.id); + } else { + capacity.write_owners.remove(&self.path); + } + drop(capacity); self.replacement = None; self.finish_mutation(None); } pub(super) fn finish_delete(mut self, deleted_size: u64) { - self.finish_mutation(Some(deleted_size)); + let unaccounted = self + .capacity + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reserved_spools + .remove(&self.path) + .is_some(); + self.finish_mutation(Some(if unaccounted { 0 } else { deleted_size })); self.replacement = None; } @@ -134,11 +174,17 @@ impl CapacityMutationReservation { capacity.active_mutations.remove(&self.path); if capacity.initialized && capacity.cache_path == self.cache_path { if let Some(deleted_size) = deleted_size { + capacity.write_owners.remove(&self.path); + capacity.abandoned_writes.remove(&self.path); + if super::revisions::is_revision_cache_path(&self.path) { + capacity.revision_bytes = capacity.revision_bytes.saturating_sub(deleted_size); + } capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size); let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size)); } } + update_revision_metrics(&capacity); capacity.revision = Arc::new(CapacityRevision); drop(capacity); self.changed.notify_waiters(); @@ -152,18 +198,22 @@ impl Drop for CapacityMutationReservation { return; } capacity.active_mutations.remove(&self.path); + if !self.filesystem_mutation_started && self.transferred_bytes > 0 { + if let Some((_, bytes)) = + self.write_budget.as_ref().and_then(|budget| capacity.revision_reservations.get_mut(&budget.id)) + { + *bytes = bytes.saturating_add(self.transferred_bytes); + } + } let revision_changed = if self.filesystem_mutation_started { capacity.initialized = false; true } else if capacity.initialized && capacity.cache_path == self.cache_path { if let Some((old_size, new_size)) = self.replacement.take() { - capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size); + let growth = new_size.saturating_sub(old_size); + capacity.total_bytes = capacity.total_bytes.saturating_sub(growth); let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - store_session_bytes( - &mut capacity, - &self.session_component, - session_size.saturating_sub(new_size).saturating_add(old_size), - ); + store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(growth)); true } else { false @@ -342,13 +392,36 @@ impl HlsSegmentCache { notified.await; continue; }; - let (total_bytes, session_bytes, revision_bytes) = scan_committed_cache_usage(cache_path).await?; + let (temp_revision, mut excluded) = { + let temps = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let excluded: std::collections::HashSet<_> = temps + .active_files + .iter() + .filter(|path| super::lifecycle::is_temp_cache_file(path)) + .cloned() + .collect(); + (Arc::clone(&temps.revision), excluded) + }; + excluded.extend( + self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reserved_spools.keys().cloned(), + ); + #[cfg(test)] + self.startup_write_checkpoint(super::StartupWritePhase::AfterInventorySnapshot).await?; + let (total_bytes, session_bytes, revision_bytes) = + scan_cache_usage_excluding(cache_path, &excluded).await?; if self.cache_path_snapshot().path != cache_path { return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan")); } + // Registration precedes file creation. Changes after this check cannot + // have affected the completed scan; no nested temp/capacity locks are needed. + let temps_changed = !Arc::ptr_eq( + &self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).revision, + &temp_revision, + ); let scan_invalidated = { let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !Arc::ptr_eq(&capacity.revision, &revision) + if temps_changed + || !Arc::ptr_eq(&capacity.revision, &revision) || capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) { true @@ -358,6 +431,18 @@ impl HlsSegmentCache { capacity.total_bytes = total_bytes; capacity.session_bytes = session_bytes; capacity.revision_bytes = revision_bytes; + let sessions: std::collections::HashSet<_> = capacity.session_bytes.keys().cloned().collect(); + let active_owners: std::collections::HashSet<_> = + capacity.revision_reservations.keys().copied().collect(); + capacity.write_owners.retain(|path, owner| { + active_owners.contains(owner) + || path.strip_prefix(cache_path).ok().and_then(|path| path.components().next()).is_some_and( + |session| sessions.contains(session.as_os_str().to_string_lossy().as_ref()), + ) + }); + let owners = capacity.write_owners.clone(); + capacity.abandoned_writes.retain(|path, (owner, _)| owners.get(path) == Some(owner)); + update_revision_metrics(&capacity); false } }; @@ -407,6 +492,8 @@ impl HlsSegmentCache { session_component, replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed), }) @@ -421,9 +508,39 @@ impl HlsSegmentCache { max_cache_bytes: u64, max_session_bytes: u64, ) -> io::Result { + self.prepare_capacity_replacement_with_budget( + cache_path, + final_path, + session_component, + new_size, + max_cache_bytes, + max_session_bytes, + None, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn prepare_capacity_replacement_with_budget( + &self, + cache_path: &Path, + final_path: &Path, + session_component: String, + new_size: u64, + max_cache_bytes: u64, + max_session_bytes: u64, + budget: Option>, + ) -> io::Result { + if budget.as_ref().is_some_and(|budget| { + !Arc::ptr_eq(&budget.capacity, &self.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + }) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation")); + } loop { self.ensure_capacity_initialized(cache_path).await?; let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await; + reservation.write_budget = budget.clone(); let old_size = match fs::metadata(final_path).await { Ok(metadata) => metadata.len(), Err(err) if err.kind() == io::ErrorKind::NotFound => 0, @@ -465,7 +582,15 @@ fn invalidate_capacity_state(capacity: &StdMutex, changed: & changed.notify_waiters(); } +#[cfg(test)] pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap, u64)> { + scan_cache_usage_excluding(cache_path, &std::collections::HashSet::new()).await +} + +async fn scan_cache_usage_excluding( + cache_path: &Path, + excluded: &std::collections::HashSet, +) -> io::Result<(u64, HashMap, u64)> { let mut total_bytes = 0_u64; let mut revision_bytes = 0_u64; let mut session_bytes = HashMap::new(); @@ -475,22 +600,37 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result< Err(err) => return Err(err), }; while let Some(root_entry) = root_entries.next_entry().await? { - if !root_entry.file_type().await?.is_dir() { - continue; + match root_entry.file_type().await { + Ok(kind) if kind.is_dir() => {} + Ok(_) => continue, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), } let session_component = root_entry.file_name().to_string_lossy().into_owned(); let mut bytes = 0_u64; let mut pending_dirs = vec![root_entry.path()]; while let Some(dir) = pending_dirs.pop() { - let mut entries = fs::read_dir(dir).await?; + let mut entries = match fs::read_dir(dir).await { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; while let Some(entry) = entries.next_entry().await? { - let file_type = entry.file_type().await?; + let file_type = match entry.file_type().await { + Ok(kind) => kind, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; if file_type.is_dir() { pending_dirs.push(entry.path()); } else if file_type.is_file() { let revision = is_revision_cache_path(&entry.path()); - if revision || !is_temp_cache_file(&entry.path()) { - let size = entry.metadata().await?.len(); + if !excluded.contains(&entry.path()) { + let size = match entry.metadata().await { + Ok(metadata) => metadata.len(), + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; bytes = bytes.saturating_add(size); if revision { revision_bytes = revision_bytes.saturating_add(size); @@ -506,3 +646,9 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result< } Ok((total_bytes, session_bytes, revision_bytes)) } + +pub(super) fn update_revision_metrics(capacity: &CacheCapacityState) { + if let Some(metrics) = capacity.revision_metrics.as_ref().and_then(Weak::upgrade) { + metrics.record_revision_disk_bytes(capacity.revision_bytes); + } +} diff --git a/backend/hls/src/cache/lifecycle.rs b/backend/hls/src/cache/lifecycle.rs index ca6181cc7..d27540ac2 100644 --- a/backend/hls/src/cache/lifecycle.rs +++ b/backend/hls/src/cache/lifecycle.rs @@ -1,9 +1,9 @@ use super::{ - revisions::is_revision_cache_path, staging::run_owned_cache_operation, ActiveTempFileRegistration, - CacheCapacityState, CacheInvalidationOutcome, CachePathDeletionReservation, CachePathGeneration, CachePathState, - CachedSegmentMetadata, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, - HlsCacheCapacityRevision, HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, - DEFAULT_HLS_CACHE_PATH, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, + staging::run_owned_cache_operation, ActiveTempFileRegistration, CacheCapacityState, CacheInvalidationOutcome, + CachePathDeletionReservation, CachePathGeneration, CachePathState, CachedSegmentMetadata, + CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, HlsCacheCapacityRevision, + HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, DEFAULT_HLS_CACHE_PATH, + MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, }; use log::warn; use std::{ @@ -41,6 +41,10 @@ impl HlsSegmentCache { capacity_reclaimer: StdRwLock::new(None), capacity_admission_gate: AsyncMutex::new(()), owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)), + #[cfg(test)] + startup_test_checkpoint: StdMutex::new(None), + #[cfg(test)] + delete_test_failures: StdMutex::new(std::collections::HashMap::new()), } } @@ -123,6 +127,7 @@ impl HlsSegmentCache { pub async fn delete(&self, key: &K) -> io::Result<()> { let cache_path = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&cache_path.path).await?; let path = Self::path_for_key_in(&cache_path.path, key); let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await; self.delete_with_reservation(path, reservation).await @@ -130,6 +135,7 @@ impl HlsSegmentCache { pub async fn delete_if_inactive(&self, key: &K) -> io::Result<()> { let cache_path = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&cache_path.path).await?; let path = Self::path_for_key_in(&cache_path.path, key); let reservation = self .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) @@ -142,6 +148,8 @@ impl HlsSegmentCache { path: PathBuf, reservation: CapacityMutationReservation, ) -> io::Result<()> { + #[cfg(test)] + self.check_cache_delete_test_failure(&path)?; run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move { let size = match fs::metadata(&path).await { Ok(metadata) => metadata.len(), @@ -190,9 +198,7 @@ impl HlsSegmentCache { let cache_path = self.cache_path_snapshot(); let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?; let mut deleted = 0_usize; - let mut removed_revision = false; for path in candidates { - let revision = is_revision_cache_path(&path); let Some(deletion) = self.reserve_path_deletion(&path) else { continue; }; @@ -209,14 +215,14 @@ impl HlsSegmentCache { match result { Ok(()) => { deleted = deleted.saturating_add(1); - removed_revision |= revision; } Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()), } } - if removed_revision { + if deleted > 0 { drop(self.capacity_invalidation_guard()); + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?; } Ok(deleted) } @@ -238,7 +244,8 @@ impl HlsSegmentCache { drop(deletion); result }) - .await + .await?; + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await } pub async fn delete_orphan_session_dirs( @@ -307,6 +314,9 @@ impl HlsSegmentCache { } } } + if removed > 0 { + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?; + } Ok(removed) } @@ -389,7 +399,8 @@ impl HlsSegmentCache { drop(deletion); result }) - .await + .await?; + self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await } pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result> { @@ -422,7 +433,14 @@ impl HlsSegmentCache { if !state.active_files.insert(path.to_path_buf()) { return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active")); } - Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) })) + state.revision = Arc::new(CapacityRevision); + drop(state); + self.capacity_changed.notify_waiters(); + Ok(Arc::new(ActiveTempFileRegistration { + path: path.to_path_buf(), + state: Arc::clone(&self.temp_files), + changed: Arc::clone(&self.capacity_changed), + })) } pub(super) fn reserve_path_deletion(&self, path: &Path) -> Option { diff --git a/backend/hls/src/cache/revision_capacity_tests/publication.rs b/backend/hls/src/cache/revision_capacity_tests/publication.rs index f41d09c9c..176a08b47 100644 --- a/backend/hls/src/cache/revision_capacity_tests/publication.rs +++ b/backend/hls/src/cache/revision_capacity_tests/publication.rs @@ -1,5 +1,81 @@ use super::*; +#[test] +fn temp_registration_and_last_owner_release_advance_inventory_revision() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let initial = Arc::clone(&cache.temp_files.lock().unwrap().revision); + let registration = cache.register_active_temp_path(&directory.path().join("segment.ts.tmp.test"))?; + let registered = Arc::clone(&cache.temp_files.lock().unwrap().revision); + assert!(!Arc::ptr_eq(&initial, ®istered)); + let last_owner = Arc::clone(®istration); + drop(registration); + assert!(Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision)); + drop(last_owner); + assert!(!Arc::ptr_eq(®istered, &cache.temp_files.lock().unwrap().revision)); + assert!(cache.temp_files.lock().unwrap().active_files.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn attaching_a_budget_to_shared_staging_returns_an_error_and_preserves_its_owner() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("shared-staging".into()), 0, "ts"); + let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"test".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(Arc::clone(&budget)), + ) + .await?; + let error = staged.clone().with_write_budget(None).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(tokio::fs::read(&staged.path).await?, b"test"); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20); + cache.commit_staged(&key, staged).await?; + drop(budget); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 4); + Ok(()) +} + +#[tokio::test] +async fn inventory_scan_retries_when_a_reserved_temp_file_is_registered_after_its_snapshot() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = Arc::new(HlsSegmentCache::with_cache_path(directory.path())); + cache.update_cache_limits(60, 60); + let key = SegmentCacheKey::new(ProxySessionId("inventory-race".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 20]).await?; + let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?); + cache.invalidate_capacity_accounting(); + let (reached, resume) = cache.pause_startup_write(StartupWritePhase::AfterInventorySnapshot); + let scan_cache = Arc::clone(&cache); + let id = key.proxy_session_id().clone(); + let scan = tokio::spawn(async move { scan_cache.capacity_usage(&id).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), reached).await.unwrap().unwrap(); + let staged = cache + .stage_temp_with_budget( + &key, + &[2; 20][..], + Instant::now() + std::time::Duration::from_secs(2), + Some(Arc::clone(&budget)), + ) + .await?; + resume.send(()).unwrap(); + let usage = tokio::time::timeout(std::time::Duration::from_secs(2), scan).await.unwrap().unwrap()?; + cache.remove_staged(staged).await?; + drop(budget); + assert_eq!(usage.global_bytes, 20, "reserved staging bytes must not enter the committed inventory"); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20); + let next = cache.reserve_revision_peak(&key, 30).await?; + drop(next); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + Ok(()) +} + #[tokio::test] async fn revision_peak_and_legacy_cache_share_the_global_and_session_quota() -> io::Result<()> { let directory = tempfile::tempdir()?; @@ -54,3 +130,375 @@ async fn processed_revision_retries_and_ranges_preserve_bytes_across_cache_repla assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0); Ok(()) } + +// These are the two real disk-cache transitions from debug/bug.log. Sparse +// fixtures preserve the exact byte counts without allocating a GiB of RAM/disk. +#[tokio::test] +async fn logged_normal_commit_consumes_its_own_peak_reservation() -> io::Result<()> { + logged_peak_case(9_994_832).await +} + +#[tokio::test] +async fn logged_revision_publication_consumes_its_remaining_peak_reservation() -> io::Result<()> { + logged_peak_case(8_052_416).await +} + +async fn logged_peak_case(size: u64) -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(1_073_741_824, 1_073_741_824); + let key = SegmentCacheKey::new(ProxySessionId("logged-session".into()), 84, "ts"); + let existing = SegmentCacheKey::new(key.proxy_session_id().clone(), 0, "ts"); + let path = cache.object_path(&existing); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + let file = tokio::fs::File::create(path).await?; + file.set_len(1_040_683_024).await?; + drop(file); + let reservation = std::sync::Arc::new(cache.reserve_revision_peak(&key, 3 * size).await?); + let staged_path = cache.object_path(&key).with_extension("ts.tmp.regression"); + let file = tokio::fs::File::create(&staged_path).await?; + file.set_len(size).await?; + drop(file); + let staged = cache.adopt_staged_file(staged_path, size)?.with_write_budget(Some(reservation.clone()))?; + let committed = cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 84, crate::SegmentRevisionKind::Processed)?; + let published = cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &committed.path, + Instant::now() + std::time::Duration::from_secs(5), + Some(reservation.clone()), + ) + .await?; + assert_eq!(published.size, size); + drop(reservation); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 1_040_683_024 + 2 * size); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + Ok(()) +} + +#[tokio::test] +async fn processed_reservation_release_wakes_before_and_after_wait_registration() -> io::Result<()> { + for registered in [false, true] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("release".into()), 0, "ts"); + let reservation = cache.reserve_revision_peak(&key, 30).await?; + assert!(!reservation.unaccounted_writes); + let revision = cache.capacity_revision(); + let waiting = cache.wait_for_capacity_change(&revision); + tokio::pin!(waiting); + if registered { + assert!(futures::poll!(waiting.as_mut()).is_pending()); + } + drop(reservation); + tokio::time::timeout(std::time::Duration::from_secs(1), waiting) + .await + .expect("releasing only a processed reservation must wake the waiter"); + } + Ok(()) +} + +#[tokio::test] +async fn concurrent_peak_owners_cannot_consume_each_others_session_or_global_budget() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = std::sync::Arc::new(HlsSegmentCache::with_cache_path(directory.path())); + cache.update_cache_limits(180, 120); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(4)); + let mut tasks = Vec::new(); + for (session, seq) in [("one", 0), ("one", 1), ("two", 0)] { + let cache = cache.clone(); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + let key = SegmentCacheKey::new(ProxySessionId(session.into()), seq, "ts"); + let peak = std::sync::Arc::new(cache.reserve_revision_peak(&key, 60).await?); + barrier.wait().await; + let staged = cache + .stage_temp_with_budget( + &key, + &[7; 20][..], + Instant::now() + std::time::Duration::from_secs(2), + Some(peak.clone()), + ) + .await?; + let metadata = cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let owner = store.create(key.proxy_session_id().clone(), seq, crate::SegmentRevisionKind::Processed)?; + cache + .publish_processed_revision_with_budget( + &owner.revision().key, + &metadata.path, + Instant::now() + std::time::Duration::from_secs(2), + Some(peak), + ) + .await?; + Ok::<_, io::Error>(()) + })); + } + barrier.wait().await; + for task in tasks { + task.await??; + } + let usage = cache.capacity_usage(&ProxySessionId("one".into())).await?; + assert_eq!(usage, HlsCacheCapacityUsage { session_bytes: 80, global_bytes: 120 }); + assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("one".into())), 0); + assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("two".into())), 0); + Ok(()) +} + +#[tokio::test] +async fn old_write_cleanup_cannot_delete_a_newer_fetch_attempt() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("replace".into()), 0, "ts"); + let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"old".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let new = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"new".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(new.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let error = cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &cache.object_path(&key), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await + .expect_err("an old startup attempt cannot publish a replacement owned by a newer attempt"); + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + assert!(cache.metadata(&revision.revision().key).await?.is_none()); + cache.delete_write_owned(&key, &old).await?; + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new"); + cache.delete_write_owned(&key, &new).await?; + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0); + Ok(()) +} + +#[tokio::test] +async fn pinned_revisions_remain_accounted_after_inventory_reconciliation_and_deletion() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let metrics = std::sync::Arc::new(crate::HlsCacheMetrics::default()); + cache.install_revision_metrics(&metrics); + let key = SegmentCacheKey::new(ProxySessionId("pinned-scan".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 20]).await?; + let store = crate::SegmentRevisionStore::default(); + let owner = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let pin = cache.pin_revision_file(&cache.object_path(&owner.revision().key))?; + *owner.revision().file_pin.lock().map_err(|_| io::Error::other("file pin"))? = Some(pin); + cache + .publish_processed_revision( + &owner.revision().key, + &cache.object_path(&key), + Instant::now() + std::time::Duration::from_secs(2), + ) + .await?; + drop(cache.capacity_invalidation_guard()); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 40); + assert_eq!(metrics.snapshot().revision_disk_bytes, cache.scan_revision_disk_bytes().await?); + drop(owner); + cache.delete_session_dir(key.proxy_session_id()).await?; + assert_eq!(metrics.snapshot().revision_disk_bytes, 0); + assert_eq!(cache.scan_revision_disk_bytes().await?, 0); + Ok(()) +} + +#[tokio::test] +async fn failed_partial_commit_cleanup_retries_without_deleting_a_replacement() -> io::Result<()> { + for replace in [false, true] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let key = SegmentCacheKey::new(ProxySessionId("cleanup-retry".into()), 0, "ts"); + let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"old".as_slice(), + Instant::now() + std::time::Duration::from_secs(2), + Some(old.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + cache.fail_next_cache_deletes(cache.object_path(&key), 1); + assert!(cache.delete_write_owned(&key, &old).await.is_err()); + drop(old); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 3); + if replace { + cache.write_bytes_and_commit(&key, b"new").await?; + } + cache.retry_abandoned_write_deletions().await; + if replace { + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new"); + } else { + assert!(cache.metadata(&key).await?.is_none()); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0); + } + cache.retry_abandoned_write_deletions().await; + } + Ok(()) +} + +#[tokio::test] +async fn cancelled_staging_keeps_its_peak_until_pending_file_io_is_joined() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(20, 20); + let key = SegmentCacheKey::new(ProxySessionId("pending-io".into()), 0, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let path = cache.object_path(&key).with_extension("ts.tmp.pending"); + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + let staged = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(budget.clone()))?; + let writer = staged.track_writer(tokio::fs::File::create(&path).await?); + let mut in_flight_io = writer.lock().await; + in_flight_io.as_mut().unwrap().write_all(b"pending").await?; + drop(budget); + let revision = cache.capacity_revision(); + drop(staged); // Models cancellation while the producer still owns its I/O handle. + let released = cache.wait_for_capacity_change(&revision); + tokio::pin!(released); + assert!(futures::poll!(released.as_mut()).is_pending()); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20); + drop(in_flight_io); + tokio::time::timeout(std::time::Duration::from_secs(2), released).await.unwrap(); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert!(!tokio::fs::try_exists(path).await?); + assert!(!cache.has_active_temp_files()); + Ok(()) +} + +#[tokio::test] +async fn cancelled_revision_link_keeps_its_peak_until_the_owned_link_and_unlink_finish() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(20, 20); + let key = SegmentCacheKey::new(ProxySessionId("pending-link".into()), 0, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?); + let staged = cache + .stage_temp_with_budget( + &key, + b"0123456789".as_slice(), + Instant::now() + std::time::Duration::from_secs(5), + Some(budget.clone()), + ) + .await?; + cache.commit_staged(&key, staged).await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink); + let source = cache.object_path(&key); + let publish = cache.publish_processed_revision_with_budget( + &revision.revision().key, + &source, + Instant::now() + std::time::Duration::from_secs(5), + Some(budget), + ); + let mut publish = Box::pin(publish); + tokio::select! { + entered = entered => entered.unwrap(), + result = &mut publish => panic!("publication bypassed its link checkpoint: {result:?}"), + } + drop(publish); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + let token = cache.capacity_revision(); + resume.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), cache.wait_for_capacity_change(&token)).await.unwrap(); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 10); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + assert!(!cache.has_active_temp_files()); + assert!(cache.metadata(&revision.revision().key).await?.is_none()); + assert_eq!(tokio::fs::read(source).await?, b"0123456789"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_cache_replacement_cannot_change_a_processed_publication_source_or_its_accounted_size( +) -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(64, 64); + let key = SegmentCacheKey::new(ProxySessionId("publication-source".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, b"old").await?; + let store = crate::SegmentRevisionStore::default(); + let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let source = cache.object_path(&key); + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink); + let publication = cache.publish_processed_revision( + &revision.revision().key, + &source, + Instant::now() + std::time::Duration::from_secs(5), + ); + tokio::pin!(publication); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut publication => panic!("publication bypassed the source checkpoint: {result:?}"), + } + let replacement = cache.write_bytes_and_commit(&key, b"larger-new-cache-body"); + tokio::pin!(replacement); + assert!(futures::poll!(replacement.as_mut()).is_pending()); + resume.send(()).unwrap(); + let (published, replaced) = + tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(publication, replacement) }) + .await + .expect("source lease releases before publication commit reacquires admission"); + let published = published?; + let replaced = replaced?; + assert_eq!(published.size, 3); + assert_eq!(tokio::fs::read(published.path).await?, b"old"); + assert_eq!(tokio::fs::read(replaced.path).await?, b"larger-new-cache-body"); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, replaced.size + 3); + assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + Ok(()) +} + +#[tokio::test] +async fn shrinking_replacement_keeps_old_bytes_charged_until_rename_finishes() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(110, 110); + let key = SegmentCacheKey::new(ProxySessionId("shrinking".into()), 0, "ts"); + let other = SegmentCacheKey::new(ProxySessionId("other-session".into()), 0, "ts"); + cache.write_bytes_and_commit(&key, &[1; 100]).await?; + let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeCacheRename); + let replacement = cache.write_bytes_and_commit(&key, &[2; 10]); + tokio::pin!(replacement); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut replacement => panic!("replacement bypassed the rename checkpoint: {result:?}"), + } + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [1; 100]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 100); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + assert!(cache.reserve_revision_peak(&key, 1).await.is_err()); + assert!(cache.reserve_revision_peak(&other, 1).await.is_err()); + resume.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), replacement).await.unwrap()?; + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 10]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 10); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + let released = cache.reserve_revision_peak(&other, 100).await?; + drop(released); + Ok(()) +} diff --git a/backend/hls/src/cache/revisions.rs b/backend/hls/src/cache/revisions.rs index c2661068a..ae0a8a202 100644 --- a/backend/hls/src/cache/revisions.rs +++ b/backend/hls/src/cache/revisions.rs @@ -27,9 +27,12 @@ impl HlsRevisionDiskReservation { impl Drop for HlsRevisionDiskReservation { fn drop(&mut self) { let mut capacity = self.capacity.lock_unpoisoned(); - capacity.revision_reservations.remove(&self.id); + let released = capacity.revision_reservations.remove(&self.id).is_some(); + capacity.reserved_spools.retain(|_, owner| *owner != self.id); if self.unaccounted_writes { capacity.initialized = false; + } + if released || self.unaccounted_writes { capacity.revision = Arc::new(CapacityRevision); } drop(capacity); @@ -48,6 +51,14 @@ impl HlsSegmentCache { bytes: u64, ) -> io::Result { let _admission = self.capacity_admission_gate.lock().await; + self.reserve_revision_peak_inner(key, bytes).await + } + + pub(super) async fn reserve_revision_peak_inner( + &self, + key: &K, + bytes: u64, + ) -> io::Result { let root = self.cache_path_snapshot(); self.ensure_capacity_initialized(&root.path).await?; let session = key.session_path_component(); @@ -77,19 +88,20 @@ impl HlsSegmentCache { HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), )); } - let id = loop { - let id = fastrand::u64(..); - if !capacity.revision_reservations.contains_key(&id) { - break id; - } - }; + let id = + capacity.next_write_id.checked_add(1).ok_or_else(|| io::Error::other("cache write identity exhausted"))?; + capacity.next_write_id = id; capacity.revision_reservations.insert(id, (session, bytes)); capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); Ok(HlsRevisionDiskReservation { id, capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed), unaccounted_writes: false, + generation: root.generation, + cache_path: root.path, }) } @@ -101,18 +113,44 @@ impl HlsSegmentCache { } pub async fn create_revision_spool(&self, key: &K) -> io::Result<(File, HlsRevisionFilePin)> { + self.create_revision_spool_with_budget(key, None).await + } + + pub(crate) async fn create_revision_spool_with_budget( + &self, + key: &K, + budget: Option<&HlsRevisionDiskReservation>, + ) -> io::Result<(File, HlsRevisionFilePin)> { + let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await; let root = self.cache_path_snapshot(); + if budget.is_some_and(|budget| !Arc::ptr_eq(&budget.generation, &root.generation)) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision reservation root retired")); + } self.ensure_cache_root_marker_for(&root.path).await?; let path = Self::path_for_key_in(&root.path, key); let pin = self.pin_revision_file(&path)?; - let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?; - fs::create_dir_all(parent).await?; - let file = OpenOptions::new().write(true).create_new(true).open(&path).await?; - Ok((file, pin)) + let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?.to_path_buf(); + let mut mutation = self.begin_capacity_mutation(&root.path, &path, key.session_path_component()).await; + if let Some(budget) = budget { + self.register_reserved_spool(path.clone(), budget); + } + super::staging::run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "revision-create", + async move { + fs::create_dir_all(parent).await?; + let file = OpenOptions::new().write(true).create_new(true).open(&path).await?; + mutation.finish_mutation(None); + drop(commit_gate); + Ok((file, pin)) + }, + ) + .await } pub async fn delete_orphan_revision_files(&self, cutoff: SystemTime) -> io::Result { let root = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&root.path).await?; let Some(mut sessions) = skip_not_found(fs::read_dir(&root.path).await)? else { return Ok(0) }; let mut deleted = 0usize; while let Some(session) = sessions.next_entry().await? { @@ -154,23 +192,90 @@ impl HlsSegmentCache { key: &K, source: &Path, deadline: Instant, + ) -> io::Result { + self.publish_processed_revision_with_budget(key, source, deadline, None).await + } + + #[allow(clippy::too_many_lines, reason = "Keeps source ownership, peak admission and immutable linking together.")] + pub(crate) async fn publish_processed_revision_with_budget( + &self, + key: &K, + source: &Path, + deadline: Instant, + budget: Option>, ) -> io::Result { let root = self.cache_path_snapshot(); let staged_path = Self::temp_path_for_key(key, &root.path); + let size = fs::metadata(source).await?.len(); + let checks_source_owner = budget.is_some(); + let budget = match budget { + Some(budget) => budget, + None => Arc::new(self.reserve_revision_peak_reclaiming(key, size).await?), + }; + let (size, source_lease) = loop { + if Instant::now() >= deadline { + return Err(io::Error::new(io::ErrorKind::TimedOut, "revision publication timed out")); + } + self.ensure_write_budget_available(key, &budget, size).await?; + // Preserve admission -> path-mutation ordering. A replacement must + // not change the source between its size check and the hardlink. + let admission = self.capacity_admission_gate.lock().await; + let source_lease = self.begin_capacity_mutation(&root.path, source, key.session_path_component()).await; + let actual_size = fs::metadata(source).await?.len(); + let remaining = { + let capacity = self.capacity.lock_unpoisoned(); + if checks_source_owner && capacity.write_owners.get(source) != Some(&budget.id) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision source write replaced")); + } + capacity.revision_reservations.get(&budget.id).map_or(0, |(_, bytes)| *bytes) + }; + drop(admission); + if actual_size <= remaining { + break (actual_size, source_lease); + } + drop(source_lease); + self.ensure_write_budget_available(key, &budget, actual_size).await?; + }; + let budget = Some(budget); let registration = self.register_active_temp_path(&staged_path)?; - if fs::hard_link(source, &staged_path).await.is_ok() { - let size = fs::metadata(&staged_path).await?.len(); - let staged = StagedCacheObject::registered(staged_path, size, Arc::clone(&root.generation), registration); + let staged = + StagedCacheObject::registered(staged_path.clone(), size, Arc::clone(&root.generation), registration) + .with_write_budget(budget.clone())?; + let source_path = source.to_path_buf(); + #[cfg(test)] + let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeRevisionLink); + // A Tokio filesystem call can finish after its caller is cancelled. + // Retain staging and its budget until the link has a definite result. + let (staged, fallback_source) = super::staging::run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "revision-link", + async move { + #[cfg(test)] + if let Some(checkpoint) = checkpoint { + checkpoint.wait().await?; + } + let fallback = if fs::hard_link(&source_path, &staged_path).await.is_ok() { + None + } else { + // Opening under the same source lease also freezes the + // inode used by a copy fallback after replacement resumes. + Some(File::open(source_path).await?) + }; + drop(source_lease); + Ok((staged, fallback)) + }, + ) + .await?; + if let Some(file) = fallback_source { + self.remove_staged(staged).await?; + let staged = self.stage_temp_with_budget(key, file, deadline, budget).await?; self.commit_staged(key, staged).await } else { - drop(registration); - let file = File::open(source).await?; - let staged = self.stage_temp_with_deadline(key, file, deadline).await?; self.commit_staged(key, staged).await } } - /// Revision file bytes as of the last completed capacity scan. + /// Current revision file bytes after completed accounting mutations. pub fn revision_disk_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().revision_bytes } /// Walks the cache tree for the current revision file bytes. diff --git a/backend/hls/src/cache/staging.rs b/backend/hls/src/cache/staging.rs index 70938876d..146f6cbcc 100644 --- a/backend/hls/src/cache/staging.rs +++ b/backend/hls/src/cache/staging.rs @@ -34,7 +34,20 @@ impl StagedCacheObject { cache_path_generation: Arc, registration: Arc, ) -> Self { - Self { path, size, cache_path_generation, _registration: registration } + Self { + cleanup: Arc::new(super::StagedFileCleanup { + path: path.clone(), + registration: Arc::clone(®istration), + budget: None, + completed: std::sync::atomic::AtomicBool::new(false), + writer: std::sync::Mutex::new(None), + }), + path, + size, + cache_path_generation, + _registration: registration, + write_budget: None, + } } } @@ -118,9 +131,10 @@ impl HlsSegmentCache { G: Send + 'static, { let staged_path = staged.path.clone(); + let cleanup = Arc::clone(&staged.cleanup); let result = self.commit_staged_inner(key, staged, guard).await; if result.is_err() { - remove_temp_file_after_error(&staged_path, "commit").await; + cleanup.completed.store(remove_temp_file_after_error(&staged_path, "commit").await, Ordering::Release); } result } @@ -164,13 +178,14 @@ impl HlsSegmentCache { }; fs::create_dir_all(parent).await?; let reservation = self - .prepare_capacity_replacement( + .prepare_capacity_replacement_with_budget( &cache_path.path, &final_path, key.session_path_component(), staged.size, self.max_cache_bytes.load(Ordering::Acquire), self.max_session_bytes.load(Ordering::Acquire), + staged.write_budget.clone(), ) .await; let reservation = match reservation { @@ -265,18 +280,25 @@ impl HlsSegmentCache { drop(admission_gate); let staged_path = staged.path.clone(); let committed_size = staged.size; + #[cfg(test)] + let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeCacheRename); // Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This // keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem // operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow // that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation. return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move { let mut reservation = reservation; + #[cfg(test)] + if let Some(checkpoint) = checkpoint { + checkpoint.wait().await?; + } reservation.mark_filesystem_mutation_started(); if let Err(err) = fs::rename(&staged_path, &final_path).await { remove_temp_file_after_error(&staged_path, "rename").await; return Err(err); } reservation.finish_replacement(); + staged.cleanup.completed.store(true, Ordering::Release); drop(staged); drop(commit_gate); Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard)) @@ -286,11 +308,17 @@ impl HlsSegmentCache { } pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> { + staged.finish_writer().await; + #[cfg(test)] + self.check_cache_delete_test_failure(&staged.path)?; let result = match fs::remove_file(&staged.path).await { Ok(()) => Ok(()), Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), Err(err) => Err(err), }; + if result.is_ok() { + staged.cleanup.completed.store(true, Ordering::Release); + } drop(staged); result } @@ -315,6 +343,20 @@ impl HlsSegmentCache { reader: &mut R, deadline: Option, ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner_with_budget(key, reader, deadline, None).await + } + + pub(super) async fn stage_temp_inner_with_budget( + &self, + key: &K, + reader: &mut R, + deadline: Option, + budget: Option>, + ) -> io::Result where K: HlsCacheObjectKey, R: AsyncRead + Unpin, @@ -322,6 +364,10 @@ impl HlsSegmentCache { if deadline.is_some_and(|deadline| deadline <= Instant::now()) { return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } + let budget = match budget { + Some(budget) => budget, + None => Arc::new(self.reserve_revision_peak_reclaiming(key, 0).await?), + }; let cache_path = self.cache_path_snapshot(); self.ensure_cache_root_marker_for(&cache_path.path).await?; let final_path = Self::path_for_key_in(&cache_path.path, key); @@ -330,28 +376,61 @@ impl HlsSegmentCache { }; fs::create_dir_all(parent).await?; - let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?; + let (temp_path, temp_file, registration) = self.create_temp_file(key, &cache_path).await?; + let cleanup = StagedCacheObject::registered( + temp_path.clone(), + 0, + Arc::clone(&cache_path.generation), + Arc::clone(®istration), + ) + .with_write_budget(Some(Arc::clone(&budget)))?; + let writer = cleanup.track_writer(temp_file); if deadline.is_some_and(|deadline| deadline <= Instant::now()) { - drop(temp_file); - remove_temp_file_after_error(&temp_path, "expired-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "expired-write").await, Ordering::Release); return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } let copy = async { + let mut writer = writer.lock().await; + let temp_file = writer.as_mut().ok_or_else(|| io::Error::other("staging writer already closed"))?; let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); let mut limited = reader.take(max_object_bytes.saturating_add(1)); - let size = tokio::io::copy(&mut limited, &mut temp_file).await?; + let mut size = 0u64; + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = limited.read(&mut buffer).await?; + if read == 0 { + break; + } + let next_size = size.saturating_add(u64::try_from(read).map_err(io::Error::other)?); + if next_size > max_object_bytes { + return Err(cache_object_limit_error(max_object_bytes)); + } + // Admission precedes every filesystem write, including headerless + // bodies and standalone cache users without a startup transaction. + self.ensure_write_budget_available(key, &budget, next_size).await?; + temp_file.write_all(&buffer[..read]).await?; + size = next_size; + } if size > max_object_bytes { return Err(cache_object_limit_error(max_object_bytes)); } temp_file.flush().await?; - drop(temp_file); + writer.take(); Ok::(size) }; let copy_result = if let Some(deadline) = deadline { if let Ok(result) = timeout_at(deadline, copy).await { result } else { - remove_temp_file_after_error(&temp_path, "timed-out-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "timed-out-write").await, Ordering::Release); return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); } } else { @@ -359,10 +438,16 @@ impl HlsSegmentCache { }; match copy_result { Ok(size) => { - Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration)) + let mut staged = cleanup; + staged.size = size; + Ok(staged) } Err(err) => { - remove_temp_file_after_error(&temp_path, "failed-write").await; + cleanup.finish_writer().await; + cleanup + .cleanup + .completed + .store(remove_temp_file_after_error(&temp_path, "failed-write").await, Ordering::Release); Err(err) } } @@ -411,12 +496,13 @@ impl HlsSegmentCache { } } -async fn remove_temp_file_after_error(path: &Path, operation: &'static str) { +async fn remove_temp_file_after_error(path: &Path, operation: &'static str) -> bool { match fs::remove_file(path).await { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Ok(()) => true, + Err(error) if error.kind() == io::ErrorKind::NotFound => true, Err(error) => { warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind()); + false } } } diff --git a/backend/hls/src/cache/test_support.rs b/backend/hls/src/cache/test_support.rs new file mode 100644 index 000000000..36051dfe0 --- /dev/null +++ b/backend/hls/src/cache/test_support.rs @@ -0,0 +1,69 @@ +use super::HlsSegmentCache; +use crate::sync_ext::MutexExt; +use std::{ + io, + path::{Path, PathBuf}, +}; +use tokio::sync::oneshot; + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) enum StartupWritePhase { + AfterInventorySnapshot, + BeforeNormalCommit, + AfterNormalCommit, + BeforeRevisionLink, + BeforeCacheRename, + AfterPublication, +} + +pub(super) struct StartupWriteCheckpoint { + phase: StartupWritePhase, + entered: oneshot::Sender<()>, + resume: oneshot::Receiver<()>, +} + +impl StartupWriteCheckpoint { + pub(super) async fn wait(self) -> io::Result<()> { + let _ = self.entered.send(()); + self.resume.await.map_err(|_| io::Error::from_raw_os_error(28)) + } +} + +impl HlsSegmentCache { + pub(crate) fn pause_startup_write(&self, phase: StartupWritePhase) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { + let (entered, reached) = oneshot::channel(); + let (resume, released) = oneshot::channel(); + *self.startup_test_checkpoint.lock_unpoisoned() = + Some(StartupWriteCheckpoint { phase, entered, resume: released }); + (reached, resume) + } + + pub(super) fn take_startup_write_checkpoint(&self, phase: StartupWritePhase) -> Option { + let mut hook = self.startup_test_checkpoint.lock_unpoisoned(); + if hook.as_ref().is_some_and(|hook| hook.phase == phase) { + hook.take() + } else { + None + } + } + + pub(crate) async fn startup_write_checkpoint(&self, phase: StartupWritePhase) -> io::Result<()> { + if let Some(hook) = self.take_startup_write_checkpoint(phase) { + hook.wait().await?; + } + Ok(()) + } + + pub(crate) fn fail_next_cache_deletes(&self, path: PathBuf, attempts: usize) { + self.delete_test_failures.lock_unpoisoned().insert(path, attempts); + } + + pub(super) fn check_cache_delete_test_failure(&self, path: &Path) -> io::Result<()> { + let mut failures = self.delete_test_failures.lock_unpoisoned(); + if let Some(remaining) = failures.get_mut(path).filter(|remaining| **remaining > 0) { + *remaining -= 1; + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected cache deletion failure")); + } + Ok(()) + } +} diff --git a/backend/hls/src/cache/tests/admission.rs b/backend/hls/src/cache/tests/admission.rs index 9bb0758f7..c7cde451c 100644 --- a/backend/hls/src/cache/tests/admission.rs +++ b/backend/hls/src/cache/tests/admission.rs @@ -42,6 +42,8 @@ async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation() session_component: "session".to_string(), replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&cache.capacity), changed: Arc::clone(&cache.capacity_changed), }; @@ -79,6 +81,8 @@ async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() { session_component: "session".to_string(), replacement: None, filesystem_mutation_started: false, + write_budget: None, + transferred_bytes: 0, capacity: Arc::clone(&cache.capacity), changed: Arc::clone(&cache.capacity_changed), }; @@ -131,22 +135,6 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits"); cache.update_cache_limits(25, 25); - let first_staged = cache - .stage_temp_with_deadline( - &first_target, - &b"abcdefghij"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("first target stages"); - let second_staged = cache - .stage_temp_with_deadline( - &second_target, - &b"klmnopqrst"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("second target stages"); let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel(); let (resume_first_tx, resume_first_rx) = oneshot::channel(); let reclaimer = Arc::new(PausingCapacityReclaimer { @@ -158,11 +146,13 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write cache.install_capacity_reclaimer(&reclaimer); let first_cache = Arc::clone(&cache); - let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await }); + let first_task = + tokio::spawn(async move { first_cache.write_bytes_and_commit(&first_target, b"abcdefghij").await }); first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident"); let second_cache = Arc::clone(&cache); - let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await }); + let second_task = + tokio::spawn(async move { second_cache.write_bytes_and_commit(&second_target, b"klmnopqrst").await }); for _ in 0..100 { tokio::task::yield_now().await; } diff --git a/backend/hls/src/cache/write_budget.rs b/backend/hls/src/cache/write_budget.rs new file mode 100644 index 000000000..3b56b9a6c --- /dev/null +++ b/backend/hls/src/cache/write_budget.rs @@ -0,0 +1,393 @@ +//! Ownership of peak bytes across staging, rename, and immutable publication. +use super::{ + error::capacity_error, revisions::revision_reserved_bytes, ActiveTempFileRegistration, CacheCapacityPressure, + CapacityRevision, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsCacheCapacityRevision, + HlsCacheObjectKey, HlsRevisionDiskReservation, HlsSegmentCache, StagedCacheObject, +}; +use crate::sync_ext::MutexExt; +use std::{ + io, + path::PathBuf, + sync::{atomic::Ordering, Arc}, +}; +use tokio::{ + fs, + io::{AsyncRead, AsyncWriteExt}, + time::Instant, +}; + +type TrackedWriter = Arc>>; + +pub(super) struct StagedFileCleanup { + pub(super) path: PathBuf, + pub(super) registration: Arc, + pub(super) budget: Option>, + pub(super) completed: std::sync::atomic::AtomicBool, + pub(super) writer: std::sync::Mutex>, +} + +impl Drop for StagedFileCleanup { + fn drop(&mut self) { + if self.completed.load(Ordering::Acquire) { + return; + } + let path = self.path.clone(); + let registration = Arc::clone(&self.registration); + let budget = self.budget.clone(); + let writer = self.writer.lock_unpoisoned().clone(); + // Keep both the file registration and peak budget until removal completes. + // A successful rename leaves no file at this unique temporary path. + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + finish_writer(writer).await; + if let Err(error) = fs::remove_file(path).await { + if error.kind() != io::ErrorKind::NotFound { + log::warn!("HLS staging cleanup deferred: error_kind={:?}", error.kind()); + if let Some(budget) = &budget { + let mut capacity = budget.capacity.lock_unpoisoned(); + capacity.initialized = false; + capacity.revision = Arc::new(CapacityRevision); + } + } + } + drop(registration); + drop(budget); + }); + } + } +} + +async fn finish_writer(writer: Option) { + if let Some(writer) = writer { + let mut guard = writer.lock().await; + if let Some(mut file) = guard.take() { + // Tokio filesystem writes may outlive cancellation of write_all. + // Flushing the same handle joins that pending write before unlink. + let _ = file.flush().await; + } + } +} + +impl StagedCacheObject { + pub(crate) fn track_writer(&self, file: fs::File) -> TrackedWriter { + let writer = Arc::new(tokio::sync::Mutex::new(Some(file))); + *self.cleanup.writer.lock_unpoisoned() = Some(Arc::clone(&writer)); + writer + } + + pub(super) async fn finish_writer(&self) { + let writer = self.cleanup.writer.lock_unpoisoned().clone(); + finish_writer(writer).await; + } + + pub(crate) fn with_write_budget(mut self, budget: Option>) -> io::Result { + Arc::get_mut(&mut self.cleanup) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "cannot change a shared staged file's budget"))? + .budget + .clone_from(&budget); + self.write_budget = budget; + Ok(self) + } +} + +impl HlsSegmentCache { + pub(crate) async fn stage_temp_with_budget( + &self, + key: &K, + mut reader: R, + deadline: Instant, + budget: Option>, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner_with_budget(key, &mut reader, Some(deadline), budget).await + } + + pub(crate) async fn ensure_write_budget_available( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + size: u64, + ) -> io::Result<()> { + if !Arc::ptr_eq(&self.capacity, &budget.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation")); + } + let remaining = self + .capacity + .lock_unpoisoned() + .revision_reservations + .get(&budget.id) + .filter(|(owner, _)| owner == &key.session_path_component()) + .map(|(_, bytes)| *bytes) + .ok_or_else(|| io::Error::new(io::ErrorKind::Interrupted, "write reservation no longer current"))?; + if size > remaining { + if let Err(error) = self.grow_revision_peak(key, budget, size - remaining).await { + if let Some(capacity) = super::hls_cache_capacity_from_io(&error) { + let mut pressure = capacity.pressure(); + pressure.staged_bytes = size; + return Err(capacity_error( + pressure, + HlsCacheCapacityReclaimOutcome { + protected_working_set_bytes: capacity.protected_working_set_bytes, + reclaimable_bytes: capacity.reclaimable_bytes, + ..Default::default() + }, + capacity.revision().clone(), + )); + } + return Err(error); + } + } + Ok(()) + } + + pub(crate) async fn grow_revision_peak( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + additional: u64, + ) -> io::Result<()> { + let _admission = self.capacity_admission_gate.lock().await; + let root = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&root.path).await?; + let error = match self.try_grow_revision_peak(key, budget, additional) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) }; + let outcome = self + .reclaim_capacity(HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: self.object_path(key), + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: additional, + }) + .await?; + self.ensure_capacity_initialized(&root.path).await?; + self.try_grow_revision_peak(key, budget, additional).map_err(|error| { + match super::hls_cache_capacity_from_io(&error) { + Some(capacity) => capacity_error(capacity.pressure(), outcome, capacity.revision().clone()), + None => error, + } + }) + } + + fn try_grow_revision_peak( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + additional: u64, + ) -> io::Result<()> { + if !Arc::ptr_eq(&self.capacity, &budget.capacity) + || !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation) + { + return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired cache write reservation")); + } + let session = key.session_path_component(); + let mut capacity = self.capacity.lock_unpoisoned(); + let (global, local) = revision_reserved_bytes(&capacity, &session); + let global = global.saturating_add(capacity.total_bytes); + let local = local.saturating_add(capacity.session_bytes.get(&session).copied().unwrap_or_default()); + let global_limit = self.max_cache_bytes.load(Ordering::Acquire); + let local_limit = self.max_session_bytes.load(Ordering::Acquire); + if global.saturating_add(additional) > global_limit || local.saturating_add(additional) > local_limit { + return Err(capacity_error( + CacheCapacityPressure { + configured_session_bytes: local_limit, + configured_global_bytes: global_limit, + current_session_bytes: local, + current_global_bytes: global, + staged_bytes: additional, + required_session_bytes: local.saturating_add(additional).saturating_sub(local_limit), + required_global_bytes: global.saturating_add(additional).saturating_sub(global_limit), + }, + HlsCacheCapacityReclaimOutcome::default(), + HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), + )); + } + let (owner, remaining) = capacity + .revision_reservations + .get_mut(&budget.id) + .filter(|(owner, _)| owner == &session) + .ok_or_else(|| io::Error::other("write reservation no longer current"))?; + let _ = owner; + *remaining = remaining.checked_add(additional).ok_or_else(|| io::Error::other("write budget overflow"))?; + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); + Ok(()) + } + + /// Admission of a complete fresh attempt; unlike replacement admission this + /// cannot credit a leftover stable file against the next attempt's peak. + pub(crate) async fn ensure_revision_peak_capacity( + &self, + key: &K, + peak: u64, + ) -> io::Result<()> { + let reservation = self.reserve_revision_peak_reclaiming(key, peak).await?; + drop(reservation); + Ok(()) + } + + pub(crate) async fn reserve_revision_peak_reclaiming( + &self, + key: &K, + peak: u64, + ) -> io::Result { + // Reserve and reclaim under one admission gate; the reclaimer must never + // reenter admission. It only deletes already-owned files. + let _admission = self.capacity_admission_gate.lock().await; + let mut outcome = HlsCacheCapacityReclaimOutcome::default(); + for attempt in 0..2 { + match self.reserve_revision_peak_inner(key, peak).await { + Ok(reservation) => return Ok(reservation), + Err(error) => { + let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) }; + if attempt == 1 { + return Err(capacity_error(capacity.pressure(), outcome, capacity.revision().clone())); + } + outcome = self + .reclaim_capacity(HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: self.object_path(key), + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: peak, + }) + .await?; + } + } + } + unreachable!("bounded admission loop returns on second attempt") + } + + /// Deletes only the stable file installed by this exact peak owner. The + /// path mutation ticket serializes this comparison with every later rename. + pub(crate) async fn delete_write_owned( + &self, + key: &K, + budget: &HlsRevisionDiskReservation, + ) -> io::Result<()> { + let path = Self::path_for_key_in(&budget.cache_path, key); + let mutation = self.begin_capacity_mutation(&budget.cache_path, &path, key.session_path_component()).await; + let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&budget.id); + if owned { + let result = self.delete_with_reservation(path.clone(), mutation).await; + if result.is_err() { + self.capacity + .lock_unpoisoned() + .abandoned_writes + .insert(path, (budget.id, key.session_path_component())); + } + result + } else { + Ok(()) + } + } + + pub(crate) async fn retry_abandoned_write_deletions(&self) { + let pending: Vec<_> = self + .capacity + .lock_unpoisoned() + .abandoned_writes + .iter() + .take(super::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN) + .map(|(path, owner)| (path.clone(), owner.clone())) + .collect(); + let root = self.cache_path_snapshot(); + for (path, (owner, session)) in pending { + if !path.starts_with(&root.path) { + self.capacity.lock_unpoisoned().abandoned_writes.remove(&path); + continue; + } + let mutation = self.begin_capacity_mutation(&root.path, &path, session).await; + let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&owner); + if owned { + let _ = self.delete_with_reservation(path, mutation).await; + } else { + self.capacity.lock_unpoisoned().abandoned_writes.remove(&path); + } + } + } + + pub(crate) fn register_reserved_spool(&self, path: PathBuf, budget: &HlsRevisionDiskReservation) { + let mut capacity = self.capacity.lock_unpoisoned(); + capacity.write_owners.insert(path.clone(), budget.id); + capacity.reserved_spools.insert(path, budget.id); + } + + pub(crate) fn account_reserved_spool( + &self, + path: &std::path::Path, + size: u64, + budget: &HlsRevisionDiskReservation, + ) -> io::Result<()> { + let mut capacity = self.capacity.lock_unpoisoned(); + let (session, remaining) = capacity + .revision_reservations + .get_mut(&budget.id) + .ok_or_else(|| io::Error::other("spool reservation expired"))?; + let session = session.clone(); + *remaining = remaining.checked_sub(size).ok_or_else(|| io::Error::other("spool exceeds peak budget"))?; + capacity.reserved_spools.remove(path); + capacity.total_bytes = capacity.total_bytes.saturating_add(size); + *capacity.session_bytes.entry(session).or_default() += size; + capacity.revision_bytes = capacity.revision_bytes.saturating_add(size); + super::capacity::update_revision_metrics(&capacity); + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); + Ok(()) + } + + pub(crate) fn install_revision_metrics(&self, metrics: &Arc) { + let mut capacity = self.capacity.lock_unpoisoned(); + capacity.revision_metrics = Some(Arc::downgrade(metrics)); + super::capacity::update_revision_metrics(&capacity); + } + + pub(crate) fn accounted_session_bytes(&self, session: &crate::ProxySessionId) -> u64 { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + self.capacity.lock_unpoisoned().session_bytes.get(&key.session_path_component()).copied().unwrap_or_default() + } + + pub(crate) fn accounted_global_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().total_bytes } + + pub(crate) async fn session_revision_disk_bytes(&self, session: &crate::ProxySessionId) -> io::Result { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + let mut entries = match fs::read_dir(self.cache_path_snapshot().path.join(key.session_path_component())).await { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error), + }; + let mut bytes = 0u64; + while let Some(entry) = entries.next_entry().await? { + if super::revisions::is_revision_cache_path(&entry.path()) && entry.file_type().await?.is_file() { + match entry.metadata().await { + Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + } + Ok(bytes) + } + + pub(crate) fn remaining_revision_bytes(&self, session: &crate::ProxySessionId) -> u64 { + let key = super::SegmentCacheKey::new(session.clone(), 0, "ts"); + revision_reserved_bytes(&self.capacity.lock_unpoisoned(), &key.session_path_component()).1 + } +} + +impl HlsRevisionDiskReservation { + /// Resolves cleanup against this write's original root, even after handoff. + pub(crate) fn object_path(&self, key: &K) -> PathBuf { + HlsSegmentCache::path_for_key_in(&self.cache_path, key) + } +} diff --git a/backend/hls/src/gc.rs b/backend/hls/src/gc.rs index 76eca2369..876c8775e 100644 --- a/backend/hls/src/gc.rs +++ b/backend/hls/src/gc.rs @@ -37,6 +37,7 @@ pub struct HlsGarbageCollector { pending_cache_deletions: Arc>, access_leases: StdRwLock>>>, run_once_gate: AsyncMutex<()>, + revision_store: StdRwLock>>, } #[derive(Debug, Clone, Eq, PartialEq)] @@ -112,6 +113,7 @@ mod deletion; mod policy; mod reclaim; mod report; +mod revisions; mod selection; pub use collector::{build_rewrite_secret_fingerprint, exec_hls_cache_gc}; use deletion::{CacheDeletionQueueGeneration, CacheObjectDeletion}; diff --git a/backend/hls/src/gc/collector.rs b/backend/hls/src/gc/collector.rs index 1ab3a2287..36b98299b 100644 --- a/backend/hls/src/gc/collector.rs +++ b/backend/hls/src/gc/collector.rs @@ -52,6 +52,7 @@ impl HlsGarbageCollector { pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())), access_leases: StdRwLock::new(None), run_once_gate: AsyncMutex::new(()), + revision_store: StdRwLock::new(None), } } @@ -102,6 +103,11 @@ impl HlsGarbageCollector { report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?; let sessions = self.sessions.list_sessions().await; + if let Some(handle) = sessions.first() { + let id = handle.read().await.proxy_session_id.clone(); + self.cache.capacity_usage(&id).await?; + } + self.cleanup_revisions(&sessions, &mut report).await?; let mut active_session_ids = HashSet::with_capacity(sessions.len()); for session in &sessions { active_session_ids.insert(session.read().await.proxy_session_id.clone()); @@ -122,6 +128,7 @@ impl HlsGarbageCollector { session_deletions.persist(&mut report); } + self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; let mut global_deletions = self.reserve_cache_deletion_batch(); self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await; global_deletions.persist(&mut report); @@ -131,6 +138,7 @@ impl HlsGarbageCollector { self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await; } + self.cleanup_revisions(&sessions, &mut report).await?; report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); self.record_report_metrics(&report); if report.did_cleanup_or_invalidate() { diff --git a/backend/hls/src/gc/reclaim.rs b/backend/hls/src/gc/reclaim.rs index ae07f378c..c3f38164e 100644 --- a/backend/hls/src/gc/reclaim.rs +++ b/backend/hls/src/gc/reclaim.rs @@ -4,8 +4,8 @@ use super::{ selection::{ capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment, fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate, - remove_map_entry, remove_segment_entry, remove_stale_queue_entries, session_cache_size, - take_expired_transient_objects, total_sessions_cache_size, unprotected_unreferenced_map_ids, + remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries, + session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids, }, CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle, @@ -41,6 +41,8 @@ impl HlsGarbageCollector { self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await; deletion_attempt_budget = deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget)); + let sessions = self.sessions.list_sessions().await; + self.cleanup_revisions(&sessions, &mut report).await?; let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?; let required_session_bytes = request .required_session_bytes @@ -73,6 +75,30 @@ impl HlsGarbageCollector { Some(&request.target_path), release_through, ); + let head = session.segments.first_key_value(); + let reason = match head { + None => "no-candidates", + Some((seq, _)) if protected.segment_proxy_seqs.contains(seq) => "protected", + Some((_, entry)) => match entry.status { + super::SegmentCacheStatus::Ready { .. } => "reclaimable", + super::SegmentCacheStatus::CapacityDeferred { .. } if entry.origin_fetch_ref.is_some() => { + "deferred-origin-binding-retained" + } + super::SegmentCacheStatus::CapacityDeferred { .. } => "obsolete-deferred-head", + super::SegmentCacheStatus::Discovered => "not-cached", + super::SegmentCacheStatus::Queued { .. } => "queued", + super::SegmentCacheStatus::Fetching { .. } => "active-fetch", + super::SegmentCacheStatus::FailedRetryable { .. } + | super::SegmentCacheStatus::FailedPermanent { .. } => "failed-retention", + super::SegmentCacheStatus::Expired => "expired-retention", + }, + }; + log::debug!( + "HLS capacity FIFO: proxy_session={} head={:?} status={:?} blocker={reason}", + crate::safe_proxy_session_id(&session.proxy_session_id), + head.map(|(seq, _)| *seq), + head.map(|(_, entry)| &entry.status) + ); let evidence = capacity_reclamation_evidence(&session, &protected); protected_working_set_bytes = evidence.protected_working_set_bytes; reclaimable_bytes = evidence.reclaimable_bytes; @@ -95,8 +121,8 @@ impl HlsGarbageCollector { let _ = session.render_and_store_manifest(rendered_at_ms); } } else { - protected_working_set_bytes = after_pending.session_bytes; session_lock_busy = true; + log::debug!("HLS capacity selection: proxy_session={} outcome=lock-conflict protected_working_set_known=false", crate::safe_proxy_session_id(&request.proxy_session_id)); } drop(lease_guard); if session_lock_busy { @@ -112,22 +138,15 @@ impl HlsGarbageCollector { } } + self.cleanup_revisions(&sessions, &mut report).await?; let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?; let remaining_session_bytes = request .required_session_bytes .saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes)); - if remaining_session_bytes > 0 { - return Ok(HlsCacheCapacityReclaimOutcome { - reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes), - reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes), - protected_working_set_bytes, - reclaimable_bytes, - }); - } let required_global_bytes = request .required_global_bytes .saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes)); - if required_global_bytes > 0 { + if remaining_session_bytes == 0 && required_global_bytes > 0 { let sessions = self.sessions.list_sessions().await; let mut deletions = self.reserve_cache_deletion_batch(); self.collect_projected_global_reclamation( @@ -139,9 +158,13 @@ impl HlsGarbageCollector { ) .await; deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; + self.cleanup_revisions(&sessions, &mut report).await?; } let after = self.cache.capacity_usage(&request.proxy_session_id).await?; + if log::log_enabled!(log::Level::Debug) { + self.log_capacity_inventory(&request.proxy_session_id, &report).await; + } Ok(HlsCacheCapacityReclaimOutcome { reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes), reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes), @@ -150,6 +173,25 @@ impl HlsGarbageCollector { }) } + /// Diagnostic I/O cannot change the result of capacity reclamation. + pub(super) async fn log_capacity_inventory(&self, id: &crate::ProxySessionId, report: &GarbageCollectionReport) { + let revision_bytes = match self.cache.session_revision_disk_bytes(id).await { + Ok(bytes) => bytes, + Err(error) => { + log::debug!( + "HLS capacity inventory unavailable: proxy_session={} error_kind={:?}", + crate::safe_proxy_session_id(id), + error.kind() + ); + return; + } + }; + let reserved_bytes = self.cache.remaining_revision_bytes(id); + log::debug!("HLS capacity inventory: proxy_session={} revision_bytes={revision_bytes} remaining_reserved_bytes={reserved_bytes} revision_deletes_planned={} revision_deletes_succeeded={} revision_deletes_failed={} revision_reclaimed_bytes={}", + crate::safe_proxy_session_id(id), report.revision_deletions_planned, + report.revision_deletions_succeeded, report.revision_deletions_failed, report.revision_bytes_reclaimed); + } + pub(super) fn collect_session_deletions( session: &mut HlsSession, cache: &HlsSegmentCache, @@ -161,6 +203,7 @@ impl HlsGarbageCollector { report.stale_queue_entries_removed = report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); + remove_obsolete_deferred_heads(session, cache, None, None, deletions); let transient_before = session.transient.resources.len(); let protected = protected_set_for_cache_reclamation(session, cache, None, None); session.transient.prune_expired_except(now_ms, &protected.key_resource_ids); @@ -195,7 +238,8 @@ impl HlsGarbageCollector { } } - let mut session_size = session_cache_size(session); + let mut session_size = + cache.accounted_session_bytes(&session.proxy_session_id).max(session_cache_size(session)); while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { let protected = protected_set_for_cache_reclamation(session, cache, None, None); let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) @@ -247,7 +291,7 @@ impl HlsGarbageCollector { report: &mut GarbageCollectionReport, deletions: &mut CacheDeletionBatch, ) { - let mut total_size = total_sessions_cache_size(sessions).await; + let mut total_size = self.cache.accounted_global_bytes(); let mut transient_readiness_advanced = HashSet::new(); let mut skipped_transient_sessions = HashSet::new(); diff --git a/backend/hls/src/gc/report.rs b/backend/hls/src/gc/report.rs index 261ce8660..0018f2579 100644 --- a/backend/hls/src/gc/report.rs +++ b/backend/hls/src/gc/report.rs @@ -2,6 +2,11 @@ use super::ProxySessionId; #[derive(Debug, Default, Clone, Eq, PartialEq)] pub struct GarbageCollectionReport { + pub revision_deletions_planned: usize, + pub revision_deletions_succeeded: usize, + pub revision_deletions_failed: usize, + pub revision_bytes_reclaimed: u64, + pub revision_lock_conflicts: usize, pub secret_cache_invalidated: bool, pub secret_cache_invalidation_deferred: bool, pub temp_files_deleted: usize, @@ -29,7 +34,9 @@ impl GarbageCollectionReport { } pub fn did_cleanup_or_invalidate(&self) -> bool { - self.secret_cache_invalidated + self.revision_deletions_planned > 0 + || self.revision_deletions_failed > 0 + || self.secret_cache_invalidated || self.secret_cache_invalidation_deferred || self.temp_files_deleted > 0 || self.orphan_session_dirs_deleted > 0 diff --git a/backend/hls/src/gc/revisions.rs b/backend/hls/src/gc/revisions.rs new file mode 100644 index 000000000..c8e4d7aa3 --- /dev/null +++ b/backend/hls/src/gc/revisions.rs @@ -0,0 +1,141 @@ +//! Shared, bounded retirement for regular GC and projected-capacity reclamation. +use super::{GarbageCollectionReport, HlsGarbageCollector, HlsSessionHandle}; +use crate::{sync_ext::MutexExt, SegmentRevision, SegmentRevisionStore}; +use std::{ + collections::VecDeque, + io, + sync::{Arc, Weak}, +}; + +struct RetirementBatch { + store: Arc, + pending: VecDeque>, +} + +impl Drop for RetirementBatch { + fn drop(&mut self) { + // Cancellation or a failed delete preserves retry ownership, including + // all later entries in the batch which have not reached the filesystem. + for revision in self.pending.drain(..) { + let _ = self.store.requeue_retired(revision); + } + } +} + +impl HlsGarbageCollector { + pub(crate) fn install_revision_store(&self, store: &Arc) { + *self.revision_store.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::downgrade(store)); + } + + /// Caller already owns the GC gate. Never calls admission or reacquires GC. + #[allow( + clippy::too_many_lines, + reason = "One bounded lease/session snapshot followed by cancellation-safe filesystem retirement." + )] + pub(super) async fn cleanup_revisions( + &self, + sessions: &[HlsSessionHandle], + report: &mut GarbageCollectionReport, + ) -> io::Result<()> { + self.cache.retry_abandoned_write_deletions().await; + let before = ( + report.revision_deletions_planned, + report.revision_deletions_succeeded, + report.revision_deletions_failed, + report.revision_bytes_reclaimed, + ); + let mut stores: Vec<_> = self + .revision_store + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade) + .into_iter() + .collect(); + let leases = self + .access_leases + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade); + for handle in sessions { + let mut lease_guard = match &leases { + Some(leases) => { + if let Ok(guard) = leases.try_write() { + Some(guard) + } else { + report.revision_lock_conflicts += 1; + continue; + } + } + None => None, + }; + let Ok(mut session) = handle.try_write() else { + report.revision_lock_conflicts += 1; + continue; + }; + let id = session.proxy_session_id.clone(); + let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq); + let crate::HlsSession { startup, segments, .. } = &mut *session; + if let Some(startup) = startup { + startup.revisions.retain(|seq, _| segments.contains_key(seq)); + if !stores.iter().any(|store| Arc::ptr_eq(store, &startup.store)) { + stores.push(Arc::clone(&startup.store)); + } + } + if let Some(leases) = &mut lease_guard { + leases.prune_revision_bindings(&id, floor); + } + } + let mut remaining = super::MAX_CACHE_DELETE_RETRIES_PER_RUN; + for store in stores { + if remaining == 0 { + break; + } + let pending = store.retire_unpinned_limit(remaining)?; + let mut batch = RetirementBatch { store, pending: pending.into() }; + let attempts = batch.pending.len(); + for _ in 0..attempts { + let Some(revision) = batch.pending.front().cloned() else { break }; + report.revision_deletions_planned += 1; + remaining -= 1; + revision.file_pin.lock_unpoisoned().take(); + let size = self.cache.metadata(&revision.key).await?.map_or(0, |metadata| metadata.size); + match self.cache.delete_if_inactive(&revision.key).await { + Ok(()) => { + batch.pending.pop_front(); + report.revision_deletions_succeeded += 1; + report.revision_bytes_reclaimed = report.revision_bytes_reclaimed.saturating_add(size); + } + Err(error) => { + report.revision_deletions_failed += 1; + log::warn!( + "HLS revision delete deferred: proxy_session={} proxy_seq={} error_kind={:?}", + crate::safe_proxy_session_id(crate::HlsCacheObjectKey::proxy_session_id(&revision.key)), + revision.key.proxy_seq, + error.kind() + ); + // Put failed entries at the end of this bounded batch; + // its Drop requeues them after successful work finishes. + batch.pending.rotate_left(1); + } + } + } + } + self.metrics.record_revision_gc( + report.revision_deletions_planned - before.0, + report.revision_deletions_succeeded - before.1, + report.revision_deletions_failed - before.2, + report.revision_bytes_reclaimed - before.3, + ); + log::debug!( + "HLS revision GC: planned={} succeeded={} failed={} reclaimed_bytes={} lock_conflicts={}", + report.revision_deletions_planned, + report.revision_deletions_succeeded, + report.revision_deletions_failed, + report.revision_bytes_reclaimed, + report.revision_lock_conflicts + ); + Ok(()) + } +} diff --git a/backend/hls/src/gc/selection.rs b/backend/hls/src/gc/selection.rs index 4c694e8de..43ee9f261 100644 --- a/backend/hls/src/gc/selection.rs +++ b/backend/hls/src/gc/selection.rs @@ -8,15 +8,6 @@ use super::{ }; use std::{collections::HashSet, sync::Arc}; -pub(super) async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 { - let mut total_size = 0_u64; - for session in sessions { - let session = session.read().await; - total_size = total_size.saturating_add(session_cache_size(&session)); - } - total_size -} - pub(super) fn collect_projected_session_reclamation( session: &mut HlsSession, cache: &HlsSegmentCache, @@ -26,6 +17,7 @@ pub(super) fn collect_projected_session_reclamation( max_deletions: usize, deletions: &mut CacheDeletionBatch, ) { + remove_obsolete_deferred_heads(session, cache, Some(excluded_path), release_through, deletions); let mut planned_bytes = 0_u64; let mut planned_deletions = 0_usize; let mut transient_readiness_changed = false; @@ -335,3 +327,27 @@ pub(super) fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) -> session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id); Some(MapEntryDeletion { key: map.cache_key, content_length }) } + +/// Only retire an inactive deferred FIFO head once its origin binding is gone +/// and every existing working-set protection has released it. +pub(super) fn remove_obsolete_deferred_heads( + session: &mut HlsSession, + cache: &HlsSegmentCache, + excluded: Option<&std::path::Path>, + release_through: Option, + deletions: &mut CacheDeletionBatch, +) { + while deletions.has_capacity() { + let Some((&seq, entry)) = session.segments.first_key_value() else { break }; + let protected = protected_set_for_cache_reclamation(session, cache, excluded, release_through); + if entry.origin_fetch_ref.is_some() + || !matches!(entry.status, SegmentCacheStatus::CapacityDeferred { .. }) + || protected.segment_proxy_seqs.contains(&seq) + { + break; + } + if let Some(key) = remove_segment_entry(session, seq) { + deletions.push(CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration }); + } + } +} diff --git a/backend/hls/src/gc/tests.rs b/backend/hls/src/gc/tests.rs index 45ddd1f92..3aac8e52f 100644 --- a/backend/hls/src/gc/tests.rs +++ b/backend/hls/src/gc/tests.rs @@ -251,5 +251,6 @@ mod playlist; mod policy; mod publication; mod recovery; +mod revisions; mod streaming; mod transport; diff --git a/backend/hls/src/gc/tests/publication.rs b/backend/hls/src/gc/tests/publication.rs index 8f23ac22c..051327940 100644 --- a/backend/hls/src/gc/tests/publication.rs +++ b/backend/hls/src/gc/tests/publication.rs @@ -191,3 +191,104 @@ async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none()); assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some()); } + +#[test] +fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { + use crate::{ + media_reserve::{ + evaluate_lease_reserve, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackCursor, + HlsLeaseReserveInput, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + }, + recovery_timing::{HlsLeaseCutoverTiming, HlsRecoveryTriggerBudgetMs}, + terminal_tail::HlsMediaContainer, + }; + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + + let mut session = HlsSession::new(crate::HlsSessionKey::new(1, "gc-probe"), b"probe", 1_000); + let mut body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n"); + for seq in 0..9 { + write!(body, "#EXTINF:12.0,\n{seq}.ts\n").expect("test manifest writes"); + } + let OriginManifestParseOutcome::Normal(origin) = + parse_origin_media_manifest(&body, "http://origin.example/index.m3u8") + else { + panic!("normal manifest") + }; + session.apply_origin_manifest(&origin).unwrap(); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; + } + session.publishable_origin_head_proxy_seq = Some(4); + let mut cursor = HlsLeasePlaybackCursor::default(); + for seq in 0..8 { + let token = cursor.record_request_started(seq, 1_000 + seq * 10_000); + let token = cursor.bind_request_timing(token, &session.segments[&seq]); + let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000); + } + let manifest = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 71_000, + first_proxy_seq: 4, + last_proxy_seq: 6, + visible_segments: (4..7) + .map(|seq| HlsLeaseManifestSegment { + proxy_seq: seq, + duration_ms: 12_000, + uri: format!("{seq}.ts").into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }) + .collect::>() + .into(), + discontinuity_sequence: 0, + target_duration_ms: 12_000, + playlist_duration_ms: 36_000, + last_visible_media_end_ms: 36_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + let evaluate = |session: &HlsSession| { + let timeline = session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(4), 71_000); + let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { + manifest: &manifest, + cursor: &cursor, + ready_timeline: &timeline, + now_ms: 71_000, + playback_rate_guard_milli: 1_050, + recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000), + origin_path_degraded: true, + recovery_committed: false, + }); + (timeline.units.len(), reserve) + }; + let (before_units, before) = evaluate(&session); + assert_eq!(before.guaranteed_reserve_ms, 34_500); + assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0)); + // Run the actual metadata-removal function used by garbage collection. + assert!(remove_segment_entry(&mut session, 0).is_some()); + let (after_units, after) = evaluate(&session); + let still_ready = session.ready_timeline_snapshot(4, 71_000); + assert_eq!(still_ready.units.len(), 5); + assert_eq!(cursor.first_requested_proxy_seq, Some(0)); + assert_eq!(before_units, 5); + assert_eq!(after_units, 5); + assert_eq!(after, before); + let timing = + HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None); + assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500); + // Removing the entire historical prefix must keep exactly the same media clock. + for seq in 1..4 { + assert!(remove_segment_entry(&mut session, seq).is_some()); + } + assert_eq!(evaluate(&session).1, before); + // A real hole in the current READY span must still shorten the reserve. + assert!(remove_segment_entry(&mut session, 7).is_some()); + assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500); +} diff --git a/backend/hls/src/gc/tests/revisions.rs b/backend/hls/src/gc/tests/revisions.rs new file mode 100644 index 000000000..763e942a8 --- /dev/null +++ b/backend/hls/src/gc/tests/revisions.rs @@ -0,0 +1,248 @@ +use super::*; +use crate::{SegmentRevisionKind, SegmentRevisionStore}; + +#[tokio::test] +async fn diagnostic_inventory_errors_do_not_fail_capacity_reclamation() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let key = crate::SegmentCacheKey::new(id.clone(), 1, "ts"); + assert_eq!(gc.cache.capacity_usage(&id).await?.global_bytes, 0); + tokio::fs::write(gc.cache.object_path(&key).parent().unwrap(), b"not a directory").await?; + assert!(gc.cache.session_revision_disk_bytes(&id).await.is_err()); + gc.log_capacity_inventory(&id, &GarbageCollectionReport::default()).await; + let outcome = gc + .reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest { + proxy_session_id: id.clone(), + target_path: gc.cache.object_path(&key), + required_session_bytes: 0, + required_global_bytes: 0, + staged_bytes: 1, + }) + .await?; + assert_eq!(outcome, crate::HlsCacheCapacityReclaimOutcome::default()); + let occupied = crate::SegmentCacheKey::new(ProxySessionId("occupied".into()), 1, "ts"); + gc.cache.write_bytes_and_commit(&occupied, &[1; 20]).await?; + gc.cache.update_cache_limits(30, 30); + let error = gc.cache.reserve_revision_peak_reclaiming(&key, 20).await.err().unwrap(); + assert!( + crate::cache::hls_cache_capacity_from_io(&error).is_some(), + "diagnostic I/O must preserve the functional capacity error" + ); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + Ok(()) +} + +#[tokio::test] +async fn global_reclamation_retires_revisions_unpinned_by_another_sessions_segment_deletion() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, old_session) = gc_with_session(&directory).await; + let incoming = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "incoming"), b"secret", 1).await; + let old_id = old_session.read().await.proxy_session_id.clone(); + let incoming_id = incoming.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let revision = store.create(old_id.clone(), 1, SegmentRevisionKind::Processed)?; + let revision_key = revision.revision().key.clone(); + let metadata = gc.cache.write_bytes_and_commit(&revision_key, &[7; 20]).await?; + revision.revision().complete(metadata); + *revision.revision().file_pin.lock().unwrap() = + Some(gc.cache.pin_revision_file(&gc.cache.object_path(&revision_key))?); + let old_key = { + let mut session = old_session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.publishable_origin_head_proxy_seq = Some(2); + let head = session.segments.get_mut(&1).unwrap(); + head.origin_fetch_ref = None; + head.status = SegmentCacheStatus::Ready { content_length: 20, ready_at_ms: 0 }; + let key = head.cache_key.clone(); + let config = tuliprox_core::model::HlsStartupConfig::default(); + session.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 1_000, + worker: std::sync::Weak::new(), + access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + revisions: [(1, revision)].into(), + policy_fixed: true, + store, + budget: crate::ProgressiveBudgetManager::new(config), + }); + key + }; + gc.cache.write_bytes_and_commit(&old_key, &[1; 20]).await?; + gc.cache.update_cache_limits(50, 50); + assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40); + let key = crate::SegmentCacheKey::new(incoming_id.clone(), 1, "ts"); + let peak = Arc::new( + tokio::time::timeout(Duration::from_secs(2), gc.cache.reserve_revision_peak_reclaiming(&key, 40)) + .await + .expect("global reclamation must finish without a periodic GC")?, + ); + assert!(gc.cache.metadata(&old_key).await?.is_none()); + assert!(gc.cache.metadata(&revision_key).await?.is_none()); + assert_eq!(gc.cache.capacity_usage(&old_id).await?.session_bytes, 0); + assert!(old_session.read().await.startup.as_ref().unwrap().revisions.is_empty()); + let staged = gc + .cache + .stage_temp_with_budget( + &key, + &[9; 40][..], + tokio::time::Instant::now() + Duration::from_secs(2), + Some(Arc::clone(&peak)), + ) + .await?; + gc.cache.commit_staged(&key, staged).await?; + drop(peak); + assert_eq!(gc.cache.remaining_revision_bytes(&incoming_id), 0); + assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40); + assert_eq!(tokio::fs::read(gc.cache.object_path(&key)).await?, [9; 40]); + Ok(()) +} + +#[tokio::test] +async fn targeted_reclamation_deletes_only_unpinned_revisions_of_the_same_session() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let metrics = Arc::new(crate::HlsCacheMetrics::default()); + gc.cache.install_revision_metrics(&metrics); + let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?; + gc.cache.write_bytes_and_commit(&revision.revision().key, &[7; 80]).await?; + let reader = revision.clone(); + drop(revision); + gc.cache.update_cache_limits(100, 100); + let next = crate::SegmentCacheKey::new(id.clone(), 1, "ts"); + assert!(gc.cache.reserve_revision_peak_reclaiming(&next, 30).await.is_err()); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + drop(reader); + let peak = gc.cache.reserve_revision_peak_reclaiming(&next, 30).await?; + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + assert_eq!(metrics.snapshot().revision_disk_bytes, 0); + drop(peak); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + assert_eq!(gc.run_once(0).await?.revision_deletions_succeeded, 0); + Ok(()) +} + +#[tokio::test] +async fn failed_revision_deletes_preserve_bytes_and_are_retried_idempotently() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?; + let key = revision.revision().key.clone(); + gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?; + gc.cache.fail_next_cache_deletes(gc.cache.object_path(&key), 2); + drop(revision); + let first = gc.run_once(0).await?; + assert_eq!(first.revision_deletions_failed, 2); + assert_eq!(first.revision_bytes_reclaimed, 0); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + let next = gc.run_once(0).await?; + assert_eq!(next.revision_deletions_succeeded, 1); + assert_eq!(next.revision_bytes_reclaimed, 80); + assert!(!tokio::fs::try_exists(gc.cache.object_path(&key)).await?); + assert_eq!(gc.run_once(0).await?.revision_deletions_planned, 0); + Ok(()) +} + +#[tokio::test] +async fn obsolete_deferred_fifo_head_retires_without_skipping_a_protected_head() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let head = session.segments.get_mut(&1).unwrap(); + head.status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 1 }; + head.origin_fetch_ref = None; + head.access.reader_started(1); + head.cache_key.clone() + }; + gc.cache.write_bytes_and_commit(&key, b"partial-commit").await?; + let first = gc.run_once(0).await?; + assert_eq!(first.segments_deleted(), 0); + assert!(session.read().await.segments.contains_key(&1)); + session.write().await.segments.get_mut(&1).unwrap().access.reader_finished(); + let second = gc.run_once(0).await?; + assert_eq!(second.segments_deleted(), 1); + assert!(!session.read().await.segments.contains_key(&1)); + assert!(gc.cache.metadata(&key).await?.is_none()); + assert_eq!(gc.run_once(0).await?.segments_deleted(), 0); + Ok(()) +} + +#[tokio::test] +async fn revision_cleanup_skips_contended_locks_and_pressure_keeps_protection_unknown() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let leases = Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())); + gc.install_access_leases(&leases); + let lease_guard = leases.write().await; + let mut report = GarbageCollectionReport::default(); + tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report)) + .await + .expect("revision cleanup must not wait while the lease lock is held")?; + assert_eq!(report.revision_lock_conflicts, 1); + drop(lease_guard); + + let key = crate::SegmentCacheKey::new(id.clone(), 0, "ts"); + gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?; + let session_guard = session.write().await; + let mut report = GarbageCollectionReport::default(); + tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report)) + .await + .expect("revision cleanup must not reverse the lease/session lock order")?; + assert_eq!(report.revision_lock_conflicts, 1); + let outcome = tokio::time::timeout( + Duration::from_secs(2), + gc.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest { + proxy_session_id: id.clone(), + target_path: gc.cache.object_path(&key), + required_session_bytes: 1, + required_global_bytes: 0, + staged_bytes: 1, + }), + ) + .await + .expect("targeted selection must not wait for a contended session")?; + assert_eq!(outcome.protected_working_set_bytes, 0, "a lock conflict cannot claim all 80 bytes are protected"); + assert_eq!(outcome.reclaimed_session_bytes, 0); + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80); + drop(session_guard); + Ok(()) +} + +#[tokio::test] +async fn shared_revision_cleanup_retires_a_bounded_batch_and_drains_idempotently() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let (gc, session) = gc_with_session(&directory).await; + let id = session.read().await.proxy_session_id.clone(); + let store = Arc::new(SegmentRevisionStore::default()); + gc.install_revision_store(&store); + let limit = super::super::MAX_CACHE_DELETE_RETRIES_PER_RUN; + for seq in 0..limit + 3 { + let revision = store.create(id.clone(), u64::try_from(seq).unwrap(), SegmentRevisionKind::Processed)?; + gc.cache.write_bytes_and_commit(&revision.revision().key, &[7]).await?; + } + for expected in [limit, 3, 0] { + let mut report = GarbageCollectionReport::default(); + gc.cleanup_revisions(&[Arc::clone(&session)], &mut report).await?; + assert_eq!(report.revision_deletions_planned, expected); + assert_eq!(report.revision_deletions_succeeded, expected); + assert_eq!(report.revision_deletions_failed, 0); + assert_eq!(report.revision_bytes_reclaimed, u64::try_from(expected).unwrap()); + assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?); + } + assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0); + assert_eq!(gc.cache.remaining_revision_bytes(&id), 0); + Ok(()) +} diff --git a/backend/hls/src/gc/tests/transport.rs b/backend/hls/src/gc/tests/transport.rs index 4dd4c2d0a..49565d70e 100644 --- a/backend/hls/src/gc/tests/transport.rs +++ b/backend/hls/src/gc/tests/transport.rs @@ -68,8 +68,12 @@ async fn global_size_gc_subtracts_map_bytes_after_segment_removal() { session.apply_origin_manifest(&manifest).expect("manifest should map"); for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] { let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("segment fixture"); segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; } + for map in session.maps.values() { + gc.cache.write_bytes_and_commit(&map.cache_key, &[0; 10]).await.expect("map fixture"); + } session.maps.get_mut(&ProxyMapId(0)).expect("first map").status = MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; session.maps.get_mut(&ProxyMapId(1)).expect("second map").status = diff --git a/backend/hls/src/initial_strip.rs b/backend/hls/src/initial_strip.rs index 827d25938..ef17dccc5 100644 --- a/backend/hls/src/initial_strip.rs +++ b/backend/hls/src/initial_strip.rs @@ -268,3 +268,21 @@ mod tests { } } } + +#[cfg(test)] +mod short_window_tests { + use super::*; + + #[test] + fn strip_preserves_every_segment_of_an_already_short_playlist() { + for count in [1, 2] { + for mode in [HlsStripMode::Segments, HlsStripMode::Seconds] { + let strip = StripConfig { mode, value: 30 }; + let body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n{}", "#EXTINF:2.5,\nsegment.ts\n".repeat(count)); + let view = materialize_initial_hls_strip_view(&body, &strip, HlsManifestWindowPolicy::ApplyLiveWindow); + assert_eq!(view.body, body); + assert_eq!(initial_hls_strip_segments_for_durations(&strip, &vec![2_500; count]), 0); + } + } + } +} diff --git a/backend/hls/src/late_custom_response.rs b/backend/hls/src/late_custom_response.rs new file mode 100644 index 000000000..2233a80e9 --- /dev/null +++ b/backend/hls/src/late_custom_response.rs @@ -0,0 +1,168 @@ +//! Finite custom media published after the live splice deadline has elapsed. + +use crate::{ + hls_ctx::HlsCtx, + lease::{HlsAccessLeaseId, HlsTerminalTailPreparation}, + prepared_terminal_bundle::{prepared_terminal_bundle_key, HlsPreparedTerminalBundle}, + runtime_custom_tail::{ + await_prepared_standalone_bundle, snapshot_hls_runtime_custom_tail_asset, standalone_custom_asset_fingerprint, + HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsStandaloneCustomPlan, + HlsStandaloneCustomPlanKey, + }, + terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration}, + HlsAccessLeaseState, ProxySessionId, +}; +use std::{fmt, fmt::Write, sync::Arc}; +use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs}; + +/// Immutable, lease-bound custom playlist that does not consume a live splice base. +#[derive(Clone)] +pub struct HlsLateCustomResponsePlan { + pub generation: HlsTerminalTailGeneration, + pub(crate) manifest_snapshot_generation: u64, + pub(crate) key: HlsStandaloneCustomPlanKey, + proxy_session_id: ProxySessionId, + lease_id: HlsAccessLeaseId, + standalone: HlsStandaloneCustomPlan, +} + +impl HlsLateCustomResponsePlan { + pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool { + self.proxy_session_id == *proxy_session_id && self.lease_id == *lease_id + } + + pub fn manifest_body(&self) -> &str { &self.standalone.manifest_body } + + pub(crate) fn segment_bytes( + &self, + fingerprint: &str, + index: u16, + ) -> Result { + use crate::runtime_custom_tail::HlsStandaloneCustomSegmentError; + if fingerprint != standalone_custom_asset_fingerprint(self.key.asset) { + return Err(HlsStandaloneCustomSegmentError::StaleAssetFingerprint); + } + self.standalone.segment_bytes(index).ok_or(HlsStandaloneCustomSegmentError::InvalidIndex) + } +} + +impl fmt::Debug for HlsLateCustomResponsePlan { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("HlsLateCustomResponsePlan") + .field("generation", &self.generation) + .field("key", &self.key) + .finish_non_exhaustive() + } +} + +impl PartialEq for HlsLateCustomResponsePlan { + fn eq(&self, other: &Self) -> bool { + self.generation == other.generation + && self.manifest_snapshot_generation == other.manifest_snapshot_generation + && self.key == other.key + && self.proxy_session_id == other.proxy_session_id + && self.lease_id == other.lease_id + && self.standalone.manifest_body == other.standalone.manifest_body + } +} + +impl Eq for HlsLateCustomResponsePlan {} + +pub(crate) enum HlsLateCustomResponseError { + Superseded, + Unavailable(HlsTerminalTailCompatibility), +} + +pub(crate) async fn prepare_late_custom_response( + ctx: &HlsCtx, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, +) -> Result<(Arc, HlsRuntimeCustomTailAssetIdentity), HlsLateCustomResponseError> { + let lease = ctx + .hls_proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, now_ms) + .await + .ok_or(HlsLateCustomResponseError::Superseded)?; + if !matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) + || lease.playback_mode != HlsLeasePlaybackMode::Live + || lease.issued_at_ms != preparation.lease_issued_at_ms + || lease.admission_generation != preparation.expected_admission_generation + || lease.playback_cursor.cursor_generation != preparation.cursor_generation + || lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) + != Some(preparation.manifest_snapshot_generation) + { + return Err(HlsLateCustomResponseError::Superseded); + } + let snapshot = snapshot_hls_runtime_custom_tail_asset(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable) + .map_err(|error| HlsLateCustomResponseError::Unavailable(error.terminal_compatibility()))?; + let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&snapshot); + let bundle_key = prepared_terminal_bundle_key( + &snapshot.asset, + preparation.manifest_snapshot.target_duration_ms, + crate::terminal_tail::HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), bundle_key) + .await + .map_err(HlsLateCustomResponseError::Unavailable)?; + let first_proxy_seq = lease + .published_media_sequence_highwater + .unwrap_or(preparation.manifest_snapshot.last_proxy_seq) + .checked_add(1) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?; + let discontinuity_sequence = + lease.published_discontinuity_sequence_highwater.max(preparation.manifest_snapshot.discontinuity_sequence); + let path_prefix = preparation + .manifest_snapshot + .visible_segments + .first() + .and_then(|segment| { + let uri = preparation.manifest_snapshot.materialize_uri(&segment.uri); + uri.find("/hls/shared/live/").map(|offset| uri[..offset].to_owned()) + }) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::InvalidLeaseRoute))?; + let manifest_body = render_late_manifest(first_proxy_seq, discontinuity_sequence, &path_prefix, lease_id, &bundle)?; + let key = HlsStandaloneCustomPlanKey { + reason: snapshot.reason, + asset: expected_asset.media, + segment_count: bundle.key.segment_count, + }; + let standalone = HlsStandaloneCustomPlan { key, manifest_body: Arc::from(manifest_body), prepared_bundle: bundle }; + Ok(( + Arc::new(HlsLateCustomResponsePlan { + generation: HlsTerminalTailGeneration(preparation.decision_generation), + manifest_snapshot_generation: preparation.manifest_snapshot_generation, + key, + proxy_session_id: proxy_session_id.clone(), + lease_id: lease_id.clone(), + standalone, + }), + expected_asset, + )) +} + +// Keeping the layout separate makes the publication boundary testable without a live origin or cache. +fn render_late_manifest( + first_proxy_seq: u64, + discontinuity_sequence: u64, + path_prefix: &str, + lease_id: &HlsAccessLeaseId, + bundle: &HlsPreparedTerminalBundle, +) -> Result { + first_proxy_seq + .checked_add(u64::from(bundle.key.segment_count).saturating_sub(1)) + .ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?; + let fingerprint = standalone_custom_asset_fingerprint(bundle.key.asset); + let extinf = format_hls_duration_ms(bundle.source_asset_duration_ms); + let mut manifest = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{}\n#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}\n#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}\n#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n", hls_target_duration_secs(bundle.key.target_duration_ms)); + for index in 0..bundle.key.segment_count { + let _ = writeln!(manifest, "#EXTINF:{extinf},\n{path_prefix}/cvs/hls/{}/{fingerprint}/{index}.ts", lease_id.0); + } + manifest.push_str("#EXT-X-ENDLIST\n"); + Ok(manifest) +} + +#[cfg(test)] +mod tests; diff --git a/backend/hls/src/late_custom_response/tests.rs b/backend/hls/src/late_custom_response/tests.rs new file mode 100644 index 000000000..12f0a490c --- /dev/null +++ b/backend/hls/src/late_custom_response/tests.rs @@ -0,0 +1,47 @@ +use super::*; +use crate::{ + prepared_terminal_bundle::{HlsPreparedTerminalBundleKey, HlsPreparedTerminalSegment}, + terminal_tail::HlsTerminalAssetIdentity, +}; + +fn bundle() -> HlsPreparedTerminalBundle { + HlsPreparedTerminalBundle { + key: HlsPreparedTerminalBundleKey { + asset: HlsTerminalAssetIdentity { revision: 7, fingerprint: [7; 32] }, + target_duration_ms: 12_000, + segment_count: 12, + }, + source_asset_duration_ms: 10_027, + source_asset_duration_ticks_90khz: 902_400, + segments: (0..12) + .map(|index| HlsPreparedTerminalSegment { + index, + timestamp_offset_ticks_90khz: u64::from(index) * 902_400, + bytes: bytes::Bytes::from_static(b"prepared terminal segment"), + }) + .collect(), + } +} + +#[test] +fn late_manifest_keeps_target_and_sequences_and_contains_only_custom_media() { + let bundle = bundle(); + let manifest = render_late_manifest(131, 9, "/iptv", &HlsAccessLeaseId("lease".into()), &bundle) + .unwrap_or_else(|_| panic!("valid late manifest")); + assert!(manifest.contains("#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:131\n#EXT-X-DISCONTINUITY-SEQUENCE:9\n")); + assert_eq!(manifest.matches("#EXTINF:10.027,").count(), 12); + assert_eq!(manifest.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert!(manifest.contains("#EXT-X-KEY:METHOD=NONE\n")); + assert!(manifest.contains("/iptv/cvs/hls/lease/0000000000000007/11.ts")); + assert!(!manifest.contains("/hls/shared/live/")); + assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); +} + +#[test] +fn late_manifest_rejects_sequence_overflow_instead_of_reusing_live_sequences() { + let bundle = bundle(); + assert!(matches!( + render_late_manifest(u64::MAX - 1, 0, "", &HlsAccessLeaseId("lease".into()), &bundle,), + Err(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed)) + )); +} diff --git a/backend/hls/src/lease.rs b/backend/hls/src/lease.rs index f7659a6b7..46c859e18 100644 --- a/backend/hls/src/lease.rs +++ b/backend/hls/src/lease.rs @@ -144,6 +144,8 @@ pub struct HlsAccessLease { pub startup_admission: HlsLeaseStartupAdmissionState, pub playback_cursor: HlsLeasePlaybackCursor, pub last_manifest_snapshot: Option, + pub(crate) published_media_sequence_highwater: Option, + pub(crate) published_discontinuity_sequence_highwater: u64, revision_bindings: Option>, progressive_startup_claimed: bool, published_transient_resource_ids: HlsPublishedTransientResourceIds, diff --git a/backend/hls/src/lease/identity.rs b/backend/hls/src/lease/identity.rs index 6fcde43a7..5908eecec 100644 --- a/backend/hls/src/lease/identity.rs +++ b/backend/hls/src/lease/identity.rs @@ -37,7 +37,9 @@ impl HlsAccessLease { HlsLeasePlaybackMode::TerminalTail(plan) => { HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation } } - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => return None, }; Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback }) } diff --git a/backend/hls/src/lease/lifecycle.rs b/backend/hls/src/lease/lifecycle.rs index 696d78b6f..d04d5cf29 100644 --- a/backend/hls/src/lease/lifecycle.rs +++ b/backend/hls/src/lease/lifecycle.rs @@ -93,7 +93,9 @@ impl HlsAccessLease { HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => { Some(HlsTerminalTailGeneration(decision_generation)) } - HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, + HlsLeasePlaybackMode::Live + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::Ended => None, }; if self.pending_terminal_protection_release.is_none() { self.pending_terminal_protection_release = release_generation; @@ -370,6 +372,24 @@ impl HlsAccessLeaseStore { Some(token) } + pub(crate) fn record_timed_segment_request_started_if_identity_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + segment: &crate::SegmentEntry, + requested_at_ms: u64, + ) -> Option { + let token = self.record_segment_request_started_if_identity_matches( + lease_id, + proxy_session_id, + expected, + segment.proxy_seq, + requested_at_ms, + )?; + Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment)) + } + pub fn record_segment_request_completed_if_identity_matches( &mut self, lease_id: &HlsAccessLeaseId, diff --git a/backend/hls/src/lease/publication.rs b/backend/hls/src/lease/publication.rs index d0a8ad68d..c3dd770a8 100644 --- a/backend/hls/src/lease/publication.rs +++ b/backend/hls/src/lease/publication.rs @@ -154,6 +154,17 @@ impl HlsAccessLeaseStore { } lease.manifest_snapshot_generation = next_generation; snapshot.snapshot_generation = next_generation; + lease.published_media_sequence_highwater = Some( + lease + .published_media_sequence_highwater + .map_or(snapshot.last_proxy_seq, |previous| previous.max(snapshot.last_proxy_seq)), + ); + let last_discontinuity_sequence = snapshot.discontinuity_sequence.saturating_add( + u64::try_from(snapshot.visible_segments.iter().filter(|segment| segment.discontinuity_before).count()) + .unwrap_or(u64::MAX), + ); + lease.published_discontinuity_sequence_highwater = + lease.published_discontinuity_sequence_highwater.max(last_discontinuity_sequence); let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; lease.last_manifest_snapshot = Some(snapshot); lease.published_transient_resource_ids = published_transient_resource_ids; diff --git a/backend/hls/src/lease/revision_bindings.rs b/backend/hls/src/lease/revision_bindings.rs index 3bc1bcb99..a76c7e3ec 100644 --- a/backend/hls/src/lease/revision_bindings.rs +++ b/backend/hls/src/lease/revision_bindings.rs @@ -30,6 +30,13 @@ impl HlsLeaseRevisionBindings { impl HlsAccessLeaseStore { pub fn prune_revision_bindings(&mut self, session: &ProxySessionId, first_retained_seq: u64) { for lease in self.by_lease_id.values_mut().filter(|lease| lease.proxy_session_id == *session) { + if let Some(publication) = + lease.last_manifest_snapshot.as_mut().and_then(|snapshot| snapshot.startup_revisions.as_mut()) + { + if publication.revisions.first_key_value().is_some_and(|(seq, _)| *seq < first_retained_seq) { + Arc::make_mut(publication).revisions.retain(|seq, _| *seq >= first_retained_seq); + } + } let Some(bindings) = &mut lease.revision_bindings else { continue }; if bindings.revisions.first_key_value().is_some_and(|(seq, _)| *seq < first_retained_seq) { Arc::make_mut(bindings).retire_while(|seq, _| seq < first_retained_seq); diff --git a/backend/hls/src/lease/runtime_policy.rs b/backend/hls/src/lease/runtime_policy.rs index a3794c35d..487cd922c 100644 --- a/backend/hls/src/lease/runtime_policy.rs +++ b/backend/hls/src/lease/runtime_policy.rs @@ -16,7 +16,9 @@ impl HlsAccessLease { .runtime_policy_revocation .clone() .map(|token| HlsRuntimePolicyRevocationOutcome::AlreadyPending { token }), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => None, } } @@ -24,9 +26,9 @@ impl HlsAccessLease { match &self.playback_mode { HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), HlsLeasePlaybackMode::Live => self.runtime_policy_revocation.as_ref().map(|token| token.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - self.runtime_policy_denial_reason - } + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => self.runtime_policy_denial_reason, } } @@ -177,13 +179,16 @@ impl HlsAccessLeaseStore { } let preserve_finite_decision = matches!( lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } ); if preserve_finite_decision { if evidence_changed { lease.runtime_policy_denial_reason = match &lease.playback_mode { HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, + HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, }; } diff --git a/backend/hls/src/lease/terminal.rs b/backend/hls/src/lease/terminal.rs index 63a202782..06d43e642 100644 --- a/backend/hls/src/lease/terminal.rs +++ b/backend/hls/src/lease/terminal.rs @@ -85,9 +85,14 @@ fn terminal_commit_existing_outcome( { Some(HlsTerminalCommitOutcome::AlreadyCommitted) } - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - Some(HlsTerminalCommitOutcome::SupersededGeneration) + HlsLeasePlaybackMode::TerminalCustomResponse(current) + if current.generation.0 == preparation.decision_generation => + { + Some(HlsTerminalCommitOutcome::AlreadyCommitted) } + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(HlsTerminalCommitOutcome::SupersededGeneration), HlsLeasePlaybackMode::Ended => Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible), } } @@ -156,6 +161,7 @@ impl HlsAccessLeaseStore { if !matches!( lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalCustomResponse(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended ) { @@ -286,6 +292,109 @@ impl HlsAccessLeaseStore { HlsTerminalCommitOutcome::Committed } + pub(crate) fn commit_terminal_custom_response_if_generation_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + plan: std::sync::Arc, + ) -> HlsTerminalCommitOutcome { + if !plan.matches_route(proxy_session_id, lease_id) + || plan.generation.0 != preparation.decision_generation + || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation + || preparation.trigger.is_runtime_policy() + { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { + return outcome; + } + if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) + { + return outcome; + } + if self.availability_evidence_can_advance().is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + lease.admission_generation = preparation.decision_generation; + lease.playback_mode = HlsLeasePlaybackMode::TerminalCustomResponse(plan); + if self.advance_availability_evidence(proxy_session_id).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + HlsTerminalCommitOutcome::Committed + } + + pub(crate) fn late_custom_segment_access( + &mut self, + lease_id: &HlsAccessLeaseId, + fingerprint: &str, + index: u16, + now_ms: u64, + ) -> Option< + Result< + crate::runtime_custom_tail::HlsStandaloneCustomSegmentAccess, + crate::runtime_custom_tail::HlsStandaloneCustomSegmentError, + >, + > { + use crate::runtime_custom_tail::{HlsStandaloneCustomSegmentAccess, HlsStandaloneSharedLeaseBinding}; + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get(lease_id)?; + let HlsLeasePlaybackMode::TerminalCustomResponse(plan) = &lease.playback_mode else { + return None; + }; + if !lease_state_allows_use(lease.state) || now_ms >= lease.valid_until_ms { + return Some(Err(crate::runtime_custom_tail::HlsStandaloneCustomSegmentError::UnknownAccessLease)); + } + Some(plan.segment_bytes(fingerprint, index).map(|bytes| HlsStandaloneCustomSegmentAccess { + terminal_generation: Some(plan.generation), + username: std::sync::Arc::from(lease.username.as_str()), + shared_lease: Some(HlsStandaloneSharedLeaseBinding { + proxy_session_id: lease.proxy_session_id.clone(), + lease_issued_at_ms: lease.issued_at_ms, + }), + bytes, + })) + } + + pub(crate) fn activate_late_custom_segment_access( + &mut self, + lease_id: &HlsAccessLeaseId, + access: &crate::runtime_custom_tail::HlsStandaloneCustomSegmentAccess, + now_ms: u64, + timing: super::HlsAccessLeaseTiming, + ) -> Option { + let binding = access.shared_lease.as_ref()?; + let generation = access.terminal_generation?; + let lease = self.by_lease_id.get(lease_id)?; + let HlsLeasePlaybackMode::TerminalCustomResponse(plan) = &lease.playback_mode else { + return None; + }; + if lease.issued_at_ms != binding.lease_issued_at_ms + || lease.username != access.username.as_ref() + || plan.generation != generation + || !plan.matches_route(&binding.proxy_session_id, lease_id) + { + return None; + } + if let super::HlsAccessLeaseActivation::Activated { lease, .. } = + self.activate_access_lease(lease_id, &binding.proxy_session_id, now_ms, timing) + { + Some(*lease) + } else { + None + } + } + pub fn terminal_tail_replay_outcome( &mut self, lease_id: &HlsAccessLeaseId, diff --git a/backend/hls/src/lease/tests/behavior.rs b/backend/hls/src/lease/tests/behavior.rs index f66f78368..f783bc7ba 100644 --- a/backend/hls/src/lease/tests/behavior.rs +++ b/backend/hls/src/lease/tests/behavior.rs @@ -286,7 +286,7 @@ fn session_follow_up_shortens_pending_lease_once() { 3_000, HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }, ); - assert!(unchanged.is_empty()); + assert_eq!(unchanged.len(), 0); let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); assert_eq!(lease.valid_until_ms, 12_000); @@ -514,7 +514,7 @@ fn bulk_lease_removal_advances_and_preserves_session_evidence_generation() { assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 2); let after_removal = store.availability_evidence_generation(&proxy_session_id); assert!(after_removal > before_removal); - assert!(store.remove_access_leases_for_session(&proxy_session_id).is_empty()); + assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 0); assert_eq!(store.availability_evidence_generation(&proxy_session_id), after_removal); } diff --git a/backend/hls/src/lease/timing.rs b/backend/hls/src/lease/timing.rs index 217552dc5..0e726ef4b 100644 --- a/backend/hls/src/lease/timing.rs +++ b/backend/hls/src/lease/timing.rs @@ -56,6 +56,8 @@ impl HlsAccessLease { revision_bindings: None, progressive_startup_claimed: false, last_manifest_snapshot: None, + published_media_sequence_highwater: None, + published_discontinuity_sequence_highwater: 0, published_transient_resource_ids: HlsPublishedTransientResourceIds::default(), published_finalized_manifest_generations: BTreeSet::new(), manifest_snapshot_generation: 0, diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index fe45135b1..2f09be4dd 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -48,6 +48,7 @@ mod gc; pub mod header_policy; mod headers; mod ids; +mod late_custom_response; mod lease; mod lifecycle; mod manager; @@ -63,6 +64,8 @@ mod master_playlist; mod media_reserve; mod observability; mod origin; +mod origin_account_runtime; +mod origin_attempt; mod origin_progress; mod paths; mod post_refresh_availability; @@ -84,6 +87,7 @@ mod segment_revision; mod segment_watchdog; mod session; mod session_store; +mod short_manifest; mod startup_observability; mod startup_policy; mod sync_ext; @@ -128,6 +132,7 @@ pub mod api { scrub_hls_origin_headers, should_remove_hls_origin_header, }, ids::{build_proxy_session_id, HlsSessionKey, ProxySessionId, HLS_ACCESS_LEASE_ID_PLACEHOLDER}, + late_custom_response::HlsLateCustomResponsePlan, lease::{ new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseTiming, HlsAccessLeaseTouch, @@ -146,6 +151,7 @@ pub mod api { derive_hls_lease_manifest_snapshot, HlsLeaseManifestSnapshotInput, HlsTransientManifestTemplate, }, map::{MapCacheStatus, MapEntry, OriginMapKey, ProxyMapId}, + map_fetcher::MapFetchContext, master_playlist::{ HlsBandwidthPersistenceOutcome, HlsMasterBandwidth, HlsMasterBandwidthSelection, HlsSingleVariantMasterPlaylist, @@ -163,6 +169,9 @@ pub mod api { HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginSource, HlsOriginSourceKind, HlsOriginWorkClass, }, + origin_account_runtime::{ + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + }, origin_progress::publication_late_after_ms, paths::{HlsMapFile, HlsSegmentFile, TransientResourceFile}, playback::{ diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index b57067902..a97661bee 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -121,6 +121,7 @@ pub struct HlsTerminalCommitRequest<'a> { /// Media and compatibility evidence required for one terminal publication kind. pub enum HlsTerminalCommitPayload { Tail { plan: Arc, media_guard: HlsTerminalCommitMediaGuard }, + CustomResponse(Arc), Unavailable(HlsTerminalTailCompatibility), UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility), } diff --git a/backend/hls/src/manager/activity.rs b/backend/hls/src/manager/activity.rs index b066156d7..a703033e4 100644 --- a/backend/hls/src/manager/activity.rs +++ b/backend/hls/src/manager/activity.rs @@ -31,11 +31,15 @@ impl HlsProxyManager { proxy_seq: u64, requested_at_ms: u64, ) -> Option { - self.access_leases.write().await.record_segment_request_started_if_identity_matches( + let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; + let mut leases = self.access_leases.write().await; + let session = session.read().await; + let segment = session.segments.get(&proxy_seq)?; + leases.record_timed_segment_request_started_if_identity_matches( lease_id, proxy_session_id, lease_identity, - proxy_seq, + segment, requested_at_ms, ) } diff --git a/backend/hls/src/manager/cleanup.rs b/backend/hls/src/manager/cleanup.rs index 2a9817f10..f3f1700f0 100644 --- a/backend/hls/src/manager/cleanup.rs +++ b/backend/hls/src/manager/cleanup.rs @@ -2,7 +2,6 @@ use super::{ safe_proxy_session_id, HlsAccessLeaseSessionSnapshot, HlsExpiredSessionMarker, HlsProxyManager, HlsSessionHandle, ProxySessionId, }; -use crate::sync_ext::MutexExt; use log::debug; use std::{ io, @@ -181,33 +180,12 @@ impl HlsProxyManager { } let report = self.gc.run_once(now_ms).await?; self.cleanup_after_garbage_collection(&report).await; - if self.revision_store.has_created_revisions() { - for handle in self.sessions.list_sessions().await { - let snapshot = { - let mut session = handle.write().await; - let super::super::HlsSession { segments, startup, proxy_session_id, .. } = &mut *session; - startup.as_mut().map(|startup| { - startup.revisions.retain(|seq, _| segments.contains_key(seq)); - (proxy_session_id.clone(), segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq)) - }) - }; - if let Some((id, floor)) = snapshot { - self.access_leases.write().await.prune_revision_bindings(&id, floor); - } - } - } if self.revision_reconciliation_pending.load(Ordering::Acquire) { let deleted = self.segment_cache.delete_orphan_revision_files(std::time::SystemTime::now()).await?; if deleted < super::super::cache::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN { self.revision_reconciliation_pending.store(false, Ordering::Release); } } - for revision in self.revision_store.retire_unpinned()? { - revision.file_pin.lock_unpoisoned().take(); - if self.segment_cache.delete_if_inactive(&revision.key).await.is_err() { - self.revision_store.requeue_retired(revision)?; - } - } if self.revision_store.has_created_revisions() { let bytes = self.segment_cache.revision_disk_bytes(); self.metrics.record_revision_disk_bytes(bytes); diff --git a/backend/hls/src/manager/config.rs b/backend/hls/src/manager/config.rs index b415011bb..c1e7e54a1 100644 --- a/backend/hls/src/manager/config.rs +++ b/backend/hls/src/manager/config.rs @@ -107,6 +107,9 @@ impl HlsProxyManager { )); segment_cache.install_capacity_reclaimer(&gc); gc.install_access_leases(&access_leases); + let revision_store = Arc::new(super::super::SegmentRevisionStore::default()); + gc.install_revision_store(&revision_store); + segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); Self { sessions, @@ -126,7 +129,7 @@ impl HlsProxyManager { Some(Arc::clone(&availability_reevaluations)), )), progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), - revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_store, revision_reconciliation_pending: AtomicBool::new(true), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), diff --git a/backend/hls/src/manager/publication.rs b/backend/hls/src/manager/publication.rs index cb0253f1b..8f6ac90c9 100644 --- a/backend/hls/src/manager/publication.rs +++ b/backend/hls/src/manager/publication.rs @@ -34,6 +34,7 @@ impl HlsTerminalCommitPayload { pub(super) fn into_parts(self) -> (HlsTerminalLeaseDecision, Option) { match self { Self::Tail { plan, media_guard } => (HlsTerminalLeaseDecision::Tail(plan), Some(media_guard)), + Self::CustomResponse(plan) => (HlsTerminalLeaseDecision::CustomResponse(plan), None), Self::Unavailable(reason) => (HlsTerminalLeaseDecision::Unavailable(reason), None), Self::UnavailableAfterOwnerFailure(reason) => { (HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), None) @@ -155,6 +156,9 @@ impl HlsProxyManager { )); segment_cache.install_capacity_reclaimer(&gc); gc.install_access_leases(&access_leases); + let revision_store = Arc::new(super::super::SegmentRevisionStore::default()); + gc.install_revision_store(&revision_store); + segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); Self { sessions, @@ -174,7 +178,7 @@ impl HlsProxyManager { Some(Arc::clone(&availability_reevaluations)), )), progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), - revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_store, revision_reconciliation_pending: AtomicBool::new(true), runtime_config: ArcSwap::from_pointee(runtime_config), transient_resources: Arc::new(TransientResourceStore::new()), diff --git a/backend/hls/src/manager/terminal.rs b/backend/hls/src/manager/terminal.rs index fea5d57db..73187ec70 100644 --- a/backend/hls/src/manager/terminal.rs +++ b/backend/hls/src/manager/terminal.rs @@ -71,9 +71,26 @@ fn evaluate_terminal_commit_authorization( if !terminal_media_requirement_is_bound_to_preparation(preparation) { return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); } + if let HlsTerminalLeaseDecision::CustomResponse(plan) = decision { + if preparation.trigger.is_runtime_policy() + || !plan.matches_route(&session.proxy_session_id, lease_id) + || plan.generation.0 != preparation.decision_generation + || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation + { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); + } + if !session.origin_control.path_is_degraded_at(now_ms) { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); + } + if now_ms < preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); + } + return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; + } let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_)) && !session.can_install_terminal_tail_protection(lease_id); let (terminal, terminal_preparation) = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::CustomResponse(_), _) => unreachable!("custom response authorized above"), (HlsTerminalLeaseDecision::Tail(plan), false) => { let prepared_key = plan.media_preparation_key(); if preparation.required_terminal_media_key != Some(prepared_key) @@ -99,13 +116,12 @@ fn evaluate_terminal_commit_authorization( ) => (HlsTerminalCutoverCapability::TailUnavailable(*reason), preparation.terminal_media_preparation), }; if matches!(decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { - if !session.origin_control.path_condition.is_degraded() { + if !session.origin_control.path_is_degraded_at(now_ms) { return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); } - // This fallback publishes no media bytes. Submission arbitration and - // the terminal precondition retain the preparation's original - // exclusive deadline, so this authorization is reachable only while - // the autonomous owner still has time to commit safely. + // This fallback publishes no media bytes. Submission arbitration gives it + // a bounded CAS budget independent of the expired media-splice deadline. + // Session, recovery and lease generations still have to match. return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; } if preparation.trigger.is_runtime_policy() { @@ -190,16 +206,61 @@ impl HlsProxyManager { self.standalone_custom_access.register(entry, now_ms); } - pub fn resolve_standalone_custom_segment( + pub async fn resolve_standalone_custom_segment( &self, lease_id: &HlsAccessLeaseId, asset_fingerprint: &str, index: u16, now_ms: u64, ) -> Result { + if let Some(access) = + self.access_leases.write().await.late_custom_segment_access(lease_id, asset_fingerprint, index, now_ms) + { + return access; + } self.standalone_custom_access.resolve(lease_id, asset_fingerprint, index, now_ms) } + /// Records a successful authorized custom-media request for its exact terminal lease generation. + pub async fn record_standalone_custom_segment_activity( + &self, + lease_id: &HlsAccessLeaseId, + access: &HlsStandaloneCustomSegmentAccess, + now_ms: u64, + ) -> super::HlsMediaActivityCommitOutcome { + use super::{HlsCriticalHandoffStateAccess, HlsMediaActivityCommitOutcome}; + if access.terminal_generation.is_none() { + return HlsMediaActivityCommitOutcome::Committed; + } + let Some(binding) = access.shared_lease.as_ref() else { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + }; + let Some(session) = self.sessions.get_by_proxy_session_id(&binding.proxy_session_id).await else { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + }; + let result = self + .with_critical_handoff_state(&session, |leases, session| { + let timing = super::HlsAccessLeaseTiming { + active_window_ms: session.account_overlap_timing().hard_active_window_ms.saturating_mul(2), + valid_window_ms: self.session_idle_timeout_ms(), + }; + let lease = leases.activate_late_custom_segment_access(lease_id, access, now_ms, timing)?; + session.mark_authorized_media_access(now_ms); + Some(lease) + }) + .await; + match result { + HlsCriticalHandoffStateAccess::Acquired(Some(lease)) => { + self.schedule_access_lease_activity(&lease).await; + self.schedule_access_lease_validity(&lease).await; + self.schedule_session_idle_for_handle(&session).await; + HlsMediaActivityCommitOutcome::Committed + } + HlsCriticalHandoffStateAccess::Acquired(None) => HlsMediaActivityCommitOutcome::StaleLeaseIdentity, + HlsCriticalHandoffStateAccess::LockBusy => HlsMediaActivityCommitOutcome::DeferredLockContention, + } + } + pub fn terminal_pending(&self) -> Arc { Arc::clone(&self.terminal_pending) } /// Cancels terminal work frozen before newly committed shared media @@ -277,7 +338,7 @@ impl HlsProxyManager { session.activity.media_readiness_generation, session.origin_control.origin_epoch, session.origin_control.last_media_progress_at_ms, - session.origin_control.path_condition.is_degraded(), + session.origin_control.path_is_degraded_at(now_ms), recovery_snapshot, ) }; @@ -470,7 +531,9 @@ impl HlsProxyManager { match command.asset_revision_guard.validate_current() { HlsTerminalAssetRevisionValidation::Current => {} HlsTerminalAssetRevisionValidation::Changed { current } => { - if command.preparation.trigger.is_runtime_policy() { + if command.preparation.trigger.is_runtime_policy() + || matches!(decision, HlsTerminalLeaseDecision::CustomResponse(_)) + { return HlsTerminalCommitAttempt::Completed(HlsTerminalCommitOutcome::SupersededGeneration); } let reason = current.asset.map_or(HlsTerminalTailCompatibility::MissingAsset, |_| { @@ -525,7 +588,7 @@ impl HlsProxyManager { let proxy_session_id = &command.key.proxy_session_id; let preparation = &command.preparation; let replay = match &command.decision { - HlsTerminalLeaseDecision::Tail(_) => { + HlsTerminalLeaseDecision::Tail(_) | HlsTerminalLeaseDecision::CustomResponse(_) => { leases.terminal_tail_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) } HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => { @@ -545,7 +608,8 @@ impl HlsProxyManager { && session.activity.media_readiness_generation == preparation.media_readiness_generation && session.origin_control.origin_epoch == preparation.origin_epoch && session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms - && hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms) + && (matches!(command.decision, HlsTerminalLeaseDecision::CustomResponse(_)) + || hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms)) { return None; } @@ -575,7 +639,9 @@ impl HlsProxyManager { }), (HlsTerminalLeaseDecision::Tail(_), true) | ( - HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), + HlsTerminalLeaseDecision::CustomResponse(_) + | HlsTerminalLeaseDecision::Unavailable(_) + | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), _, ) => None, }; @@ -595,6 +661,14 @@ impl HlsProxyManager { } } let outcome = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::CustomResponse(plan), _) => leases + .commit_terminal_custom_response_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + now_ms, + Arc::clone(plan), + ), (HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches( lease_id, proxy_session_id, diff --git a/backend/hls/src/manager/tests/support.rs b/backend/hls/src/manager/tests/support.rs index 4fd3d0a23..7055da583 100644 --- a/backend/hls/src/manager/tests/support.rs +++ b/backend/hls/src/manager/tests/support.rs @@ -254,6 +254,20 @@ pub(in crate::manager::tests) async fn live_media_fixture( let manager = HlsProxyManager::with_hls_cache_config(&config); let (session, _) = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + { + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + let OriginManifestParseOutcome::Normal(manifest) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4,\n40.ts\n", + "http://origin.example/index.m3u8", + ) else { + panic!("live media fixture parses"); + }; + let mut session = session.write().await; + session.proxy_next_seq = Some(40); + session.apply_origin_manifest(&manifest).expect("live segment timeline"); + session.segments.get_mut(&40).expect("live segment").status = + crate::SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; + } let proxy_session_id = session.read().await.proxy_session_id.clone(); let lease_id = HlsAccessLeaseId(lease_name.to_string()); let lease = access_lease(lease_name, &proxy_session_id); diff --git a/backend/hls/src/manager/tests/terminal.rs b/backend/hls/src/manager/tests/terminal.rs index bb3f74d56..d8f911359 100644 --- a/backend/hls/src/manager/tests/terminal.rs +++ b/backend/hls/src/manager/tests/terminal.rs @@ -1253,3 +1253,32 @@ async fn hls_terminal_commit_protection_capacity_has_typed_unavailable_outcome() assert_eq!(session.terminal_tail_protection_count(), HLS_TERMINAL_TAIL_PROTECTION_CAPACITY); assert!(session.terminal_tail_protection(&lease_id).is_none()); } + +#[tokio::test] +async fn unavailable_after_expired_splice_deadline_still_rejects_recovered_origin_generation() { + let (manager, session, proxy_session_id, lease_id, preparation, _) = + prepared_terminal_commit_fixture("recovered-before-unavailable").await; + let now_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + 1; + manager.terminal_commit_clock.set_fixed_now_ms(now_ms); + session.write().await.origin_control.record_media_progress(now_ms, 12_000); + let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms, + payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure( + HlsTerminalTailCompatibility::TerminalMediaNotReady, + ), + asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_for_test(None), + }); + assert_eq!(outcome, HlsTerminalCommitOutcome::RecoveryCommitted); + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("recovered lease") + .playback_mode, + HlsLeasePlaybackMode::Live + ); +} diff --git a/backend/hls/src/manifest_acceptance/cohorts.rs b/backend/hls/src/manifest_acceptance/cohorts.rs index 92677cf33..136bf245e 100644 --- a/backend/hls/src/manifest_acceptance/cohorts.rs +++ b/backend/hls/src/manifest_acceptance/cohorts.rs @@ -1,4 +1,5 @@ use super::{ + evidence::HlsManifestWindowClass, timeline::{host_local_windows_compatible, same_host_timeline_compatible, strong_anchor_overlap}, HlsAlternativeOriginCohort, HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow, HlsCandidateHostRelation, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, @@ -18,6 +19,7 @@ pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> .iter() .filter(|candidate| { candidate.host_relation == HlsCandidateHostRelation::OtherHost + && candidate.timeline_fingerprint.window_class() != HlsManifestWindowClass::Empty && candidate.resource_timeline_evidence.permits_acceptance() && candidate.switch_segment_readiness.can_be_staged() && candidate.effective_host.is_some() @@ -45,7 +47,12 @@ pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> let mut cohorts = groups.into_iter().filter_map(|samples| build_alternative_cohort(&samples, &pinned)).collect::>(); cohorts.sort_by_key(|cohort| { - (evidence_priority(cohort.evidence), std::cmp::Reverse(cohort.successful_samples), cohort.best_candidate_index) + ( + cohort.window.fingerprint.window_class(), + evidence_priority(cohort.evidence), + std::cmp::Reverse(cohort.successful_samples), + cohort.best_candidate_index, + ) }); cohorts } @@ -215,7 +222,9 @@ fn build_alternative_cohort( } else { HlsCrossHostAcceptanceEvidence::Insufficient }; - let best = samples.iter().min_by_key(|sample| (sample.manifest_fetch_elapsed_ms, sample.candidate_index))?; + let best = samples.iter().min_by_key(|sample| { + (sample.timeline_fingerprint.window_class(), sample.manifest_fetch_elapsed_ms, sample.candidate_index) + })?; let best_candidate_index = best.candidate_index; let effective_host = first.effective_host.clone()?; Some(HlsAlternativeOriginCohort { diff --git a/backend/hls/src/manifest_acceptance/episode.rs b/backend/hls/src/manifest_acceptance/episode.rs index 8e45c0500..00909bc05 100644 --- a/backend/hls/src/manifest_acceptance/episode.rs +++ b/backend/hls/src/manifest_acceptance/episode.rs @@ -218,6 +218,39 @@ impl HlsManifestAcceptanceEpisode { HlsRecoveryWorkloadBindingUpdate::Applied } + /// Reopens only the selection step of an authorized reduced retry. Holding owns + /// the selection reset; all workload and timing guards still run in `select_candidate`. + pub(crate) fn select_candidate_for_reduced_retry( + &mut self, + expected_generation: HlsManifestAcceptanceGeneration, + candidate_identity: HlsManifestRecoveryCandidateIdentity, + ) -> HlsRecoveryWorkloadBindingUpdate { + if self.generation != expected_generation { + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; + } + if !self.full_burst_completed + || self.deterministic_conflict_receipt.is_some() + || !matches!( + self.outcome, + HlsManifestAcceptanceEpisodeOutcome::Pending + | HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( + HlsManifestAcceptanceExhaustionReason::AllFailed + | HlsManifestAcceptanceExhaustionReason::NoProgress + | HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate + ) + ) + { + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; + } + let previous_outcome = self.outcome; + self.outcome = HlsManifestAcceptanceEpisodeOutcome::Pending; + let update = self.select_candidate(expected_generation, candidate_identity); + if update != HlsRecoveryWorkloadBindingUpdate::Applied { + self.outcome = previous_outcome; + } + update + } + pub fn bind_selected_candidate( &mut self, expected_generation: HlsManifestAcceptanceGeneration, diff --git a/backend/hls/src/manifest_acceptance/evaluation.rs b/backend/hls/src/manifest_acceptance/evaluation.rs index 2ff6244bf..710f32e36 100644 --- a/backend/hls/src/manifest_acceptance/evaluation.rs +++ b/backend/hls/src/manifest_acceptance/evaluation.rs @@ -1,22 +1,16 @@ use super::{ - alternative_cohorts_with_history, HlsAlternativeOriginCohort, HlsCandidateHostRelation, - HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, - HlsManifestAcceptanceInput, HlsManifestAcceptanceTrigger, HlsManifestCandidateObservation, HlsManifestCommitKind, - HlsManifestCommitPlan, HlsSwitchSegmentReadiness, + alternative_cohorts_with_history, evidence::HlsManifestWindowClass, HlsAlternativeOriginCohort, + HlsCandidateHostRelation, HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, + HlsHostLocalSequenceRelation, HlsManifestAcceptanceInput, HlsManifestAcceptanceTrigger, + HlsManifestCandidateObservation, HlsManifestCommitKind, HlsManifestCommitPlan, HlsSwitchSegmentReadiness, }; pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> HlsManifestCommitPlan { - let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations); - if let Some(pinned) = progressed_pinned { - return pinned_commit(pinned); - } - - // An unchanged pinned response is useful while reserve remains. Once the - // recovery deadline is reached it must not hide a qualified alternative. - if !input.trigger.recovery_required() { - if let Some(pinned) = unchanged_pinned { - return pinned_commit(pinned); - } + let pinned = preferred_pinned_candidate(input); + if let Some(candidate) = + pinned.filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + { + return pinned_commit(candidate); } if !input.full_burst_completed { @@ -27,32 +21,46 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl }; } - if let Some(initial) = input - .observations - .iter() - .filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::InitialBaseline - && candidate.resource_timeline_evidence.permits_acceptance() - }) - .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) - { - return pinned_commit(initial); - } - let cohorts = alternative_cohorts_with_history( input.observations, input.previous_alternative, input.current_burst_is_full_plan, ); + // Exhaust all existing evidence rules for regular windows before considering + // short fallbacks. Short observations remain available as timeline anchors. + for window_class in [HlsManifestWindowClass::Regular, HlsManifestWindowClass::Short] { + if let Some(initial) = initial_baseline_candidate(input.observations, window_class) { + return pinned_commit(initial); + } + if let Some(candidate) = pinned.filter(|_| window_class == HlsManifestWindowClass::Short) { + return pinned_commit(candidate); + } + if let Some(plan) = qualified_alternative_plan(input, &cohorts, window_class) { + return plan; + } + } + if cohorts.is_empty() { + HlsManifestCommitPlan::RejectAll + } else { + HlsManifestCommitPlan::HoldAlternative + } +} + +fn qualified_alternative_plan( + input: HlsManifestAcceptanceInput<'_>, + cohorts: &[HlsAlternativeOriginCohort], + window_class: HlsManifestWindowClass, +) -> Option { if let Some(cohort) = cohorts .iter() + .filter(|cohort| cohort.window.fingerprint.window_class() == window_class) .find(|cohort| matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. })) { - return alternative_plan( + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::AnchoredAlternative, - ); + )); } if let Some(candidate) = input @@ -62,7 +70,9 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl .observations .iter() .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::OtherHost) + .filter(|candidate| candidate.timeline_fingerprint.window_class() == window_class) .filter(|candidate| candidate.resource_timeline_evidence.permits_acceptance()) + .filter(|candidate| candidate.switch_segment_readiness.can_be_staged()) .filter(|candidate| { candidate.committed_content_anchor == HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification @@ -71,41 +81,41 @@ pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> Hl }) .flatten() { - return alternative_plan( + return Some(alternative_plan( input.observations, candidate.candidate_index, HlsManifestCommitKind::ContentVerifiedAlternative, - ); + )); } if input.trigger.recovery_required() { - if let Some(cohort) = cohorts.iter().find(|cohort| { - matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) - || cohort.consecutive_confirmed_full_bursts >= 2 - }) { - return alternative_plan( + if let Some(cohort) = + cohorts.iter().filter(|cohort| cohort.window.fingerprint.window_class() == window_class).find(|cohort| { + matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) + || cohort.consecutive_confirmed_full_bursts >= 2 + }) + { + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::AlternativeAsNewEpoch, - ); + )); } } if input.trigger == HlsManifestAcceptanceTrigger::Critical { - if let Some(cohort) = critical_single_candidate_cohort(&cohorts, input.observations) { - return alternative_plan( + if let Some(cohort) = critical_single_candidate_cohort(cohorts, input.observations) + .filter(|cohort| cohort.window.fingerprint.window_class() == window_class) + { + return Some(alternative_plan( input.observations, cohort.best_candidate_index, HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, - ); + )); } } - if cohorts.is_empty() { - HlsManifestCommitPlan::RejectAll - } else { - HlsManifestCommitPlan::HoldAlternative - } + None } fn critical_single_candidate_cohort<'a>( @@ -150,12 +160,16 @@ fn alternative_plan( fn best_pinned_candidates( observations: &[HlsManifestCandidateObservation], + full_burst_completed: bool, ) -> (Option<&HlsManifestCandidateObservation>, Option<&HlsManifestCandidateObservation>) { let mut progressed = None; let mut unchanged = None; for candidate in observations.iter().filter(|candidate| { candidate.host_relation == HlsCandidateHostRelation::PinnedHost && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.timeline_fingerprint.window_class() != HlsManifestWindowClass::Empty + && (full_burst_completed + || candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) }) { match candidate.local_sequence_relation { Some( @@ -180,8 +194,12 @@ fn best_pinned_candidates( } fn pinned_order(left: &HlsManifestCandidateObservation, right: &HlsManifestCandidateObservation) -> std::cmp::Ordering { - pinned_priority(left.local_sequence_relation) - .cmp(&pinned_priority(right.local_sequence_relation)) + left.timeline_fingerprint + .window_class() + .cmp(&right.timeline_fingerprint.window_class()) + .then_with(|| { + pinned_priority(left.local_sequence_relation).cmp(&pinned_priority(right.local_sequence_relation)) + }) .then_with(|| left.manifest_fetch_elapsed_ms.cmp(&right.manifest_fetch_elapsed_ms)) .then_with(|| left.candidate_index.cmp(&right.candidate_index)) } @@ -203,3 +221,30 @@ fn has_stageable_alternative(observations: &[HlsManifestCandidateObservation]) - && candidate.switch_segment_readiness.can_be_staged() }) } + +fn initial_baseline_candidate( + observations: &[HlsManifestCandidateObservation], + window_class: HlsManifestWindowClass, +) -> Option<&HlsManifestCandidateObservation> { + observations + .iter() + .filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::InitialBaseline + && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.timeline_fingerprint.window_class() == window_class + }) + .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) +} + +fn preferred_pinned_candidate(input: HlsManifestAcceptanceInput<'_>) -> Option<&HlsManifestCandidateObservation> { + let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations, input.full_burst_completed); + let eligible_unchanged = (!input.trigger.recovery_required()).then_some(unchanged_pinned).flatten(); + progressed_pinned + .filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + .or_else(|| { + eligible_unchanged + .filter(|candidate| candidate.timeline_fingerprint.window_class() == HlsManifestWindowClass::Regular) + }) + .or(progressed_pinned) + .or(eligible_unchanged) +} diff --git a/backend/hls/src/manifest_acceptance/evidence.rs b/backend/hls/src/manifest_acceptance/evidence.rs index 7ff92624b..f36143bc8 100644 --- a/backend/hls/src/manifest_acceptance/evidence.rs +++ b/backend/hls/src/manifest_acceptance/evidence.rs @@ -41,6 +41,24 @@ pub struct HlsManifestTimelineFingerprint { pub segment_samples: Vec, } +/// Acceptance preference shared by cohort ranking and candidate selection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum HlsManifestWindowClass { + Regular, + Short, + Empty, +} + +impl HlsManifestTimelineFingerprint { + pub(super) const fn window_class(&self) -> HlsManifestWindowClass { + match self.segment_count { + 0 => HlsManifestWindowClass::Empty, + 1 | 2 => HlsManifestWindowClass::Short, + 3.. => HlsManifestWindowClass::Regular, + } + } +} + /// Stable, window-independent technical identity of an alternative timeline. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct HlsManifestTechnicalSignature { diff --git a/backend/hls/src/manifest_acceptance/tests.rs b/backend/hls/src/manifest_acceptance/tests.rs index 582606885..c35077926 100644 --- a/backend/hls/src/manifest_acceptance/tests.rs +++ b/backend/hls/src/manifest_acceptance/tests.rs @@ -106,3 +106,4 @@ fn sliding_observation( mod episode; mod evaluation; +mod reduced_retry; diff --git a/backend/hls/src/manifest_acceptance/tests/evaluation.rs b/backend/hls/src/manifest_acceptance/tests/evaluation.rs index 05be5fb91..e26bbdc9d 100644 --- a/backend/hls/src/manifest_acceptance/tests/evaluation.rs +++ b/backend/hls/src/manifest_acceptance/tests/evaluation.rs @@ -1,5 +1,86 @@ use super::*; +#[test] +fn regular_consensus_precedes_short_timeline_anchor_without_preferred_pinned_candidate() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + let mut short = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + short.timeline_fingerprint.segment_count = 2; + short.timeline_fingerprint.segment_samples.truncate(2); + for (index, segment) in pinned.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.program_date_time_ms = Some(1_700_000_000_000 + i64::try_from(index).unwrap() * 4_000); + } + for (index, segment) in short.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.program_date_time_ms = Some(1_700_000_000_500 + i64::try_from(index).unwrap() * 4_000); + } + let regular = [ + observation(2, "origin-c", HlsCandidateHostRelation::OtherHost), + observation(3, "origin-c", HlsCandidateHostRelation::OtherHost), + ]; + for trigger in [HlsManifestAcceptanceTrigger::RecoveryRequired, HlsManifestAcceptanceTrigger::Critical] { + assert_eq!( + evaluate(&[pinned.clone(), short.clone(), regular[0].clone(), regular[1].clone()], trigger), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 2, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); + } + assert_eq!( + evaluate(&[pinned.clone(), short.clone(), regular[0].clone()], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); + assert_eq!( + evaluate(&[pinned, short, regular[0].clone(), regular[1].clone()], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); +} + +#[test] +fn regular_content_verification_precedes_short_anchor_and_unstageable_regular_does_not_block_fallback() { + let mut short = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + short.timeline_fingerprint.segment_count = 2; + short.timeline_fingerprint.segment_samples.truncate(2); + short.evidence = HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments: 2 }; + let mut regular = observation(1, "origin-c", HlsCandidateHostRelation::OtherHost); + regular.committed_content_anchor = HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification; + assert_eq!( + evaluate(&[short.clone(), regular.clone()], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::ContentVerifiedAlternative, + } + ); + regular.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; + assert_eq!( + evaluate(&[short, regular], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); +} + +#[test] +fn empty_alternative_cannot_be_qualified_by_anchor_or_consensus() { + let mut empty = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + empty.timeline_fingerprint.segment_count = 0; + empty.timeline_fingerprint.segment_samples.clear(); + empty.evidence = HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments: 2 }; + let mut other = empty.clone(); + other.candidate_index = 1; + assert_eq!( + evaluate(&[empty, other], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::RejectAll + ); +} + #[test] fn forward_media_sequence_does_not_override_published_resource_replay() { let mut replay = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); @@ -342,7 +423,7 @@ fn conflicting_reduced_follow_up_cannot_reuse_previous_full_burst() { let mut conflict = observation(0, "origin-c", HlsCandidateHostRelation::OtherHost); conflict.timeline_fingerprint = fingerprint(20); - assert!(alternative_cohorts_with_history(&[conflict], Some(&first), false).is_empty()); + assert_eq!(alternative_cohorts_with_history(&[conflict], Some(&first), false).len(), 0); } #[test] @@ -399,3 +480,83 @@ fn critical_mode_rejects_ambiguous_single_sample_cohorts() { HlsManifestCommitPlan::HoldAlternative ); } + +#[test] +fn short_initial_and_pinned_candidates_require_a_complete_burst() { + for relation in [HlsCandidateHostRelation::InitialBaseline, HlsCandidateHostRelation::PinnedHost] { + for count in [1, 2] { + let mut candidate = observation(0, "origin-a", relation); + candidate.timeline_fingerprint.segment_count = count; + candidate.timeline_fingerprint.segment_samples.truncate(usize::try_from(count).unwrap()); + assert_eq!( + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: false, + current_burst_is_full_plan: true, + trigger: HlsManifestAcceptanceTrigger::Observe, + previous_alternative: None, + observations: std::slice::from_ref(&candidate), + }), + HlsManifestCommitPlan::RejectAll + ); + assert_eq!( + evaluate(&[candidate], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); + } + } +} + +#[test] +fn eligible_regular_window_wins_over_faster_short_candidate_in_same_origin_class() { + for relation in [HlsCandidateHostRelation::InitialBaseline, HlsCandidateHostRelation::PinnedHost] { + let mut short = observation(0, "origin-a", relation); + short.timeline_fingerprint.segment_count = 1; + short.manifest_fetch_elapsed_ms = 1; + let mut regular = observation(1, "origin-a", relation); + regular.manifest_fetch_elapsed_ms = 100; + assert_eq!( + evaluate(&[short.clone(), regular.clone()], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } + ); + regular.resource_timeline_evidence = HlsResourceTimelineEvidence::ReplayOnly; + assert_eq!( + evaluate(&[short, regular], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); + } +} + +#[test] +fn empty_initial_candidates_are_not_qualified_by_a_complete_burst() { + let mut empty = observation(0, "origin-a", HlsCandidateHostRelation::InitialBaseline); + empty.timeline_fingerprint.segment_count = 0; + empty.timeline_fingerprint.segment_samples.clear(); + assert_eq!(evaluate(&[empty], HlsManifestAcceptanceTrigger::Observe), HlsManifestCommitPlan::RejectAll); +} + +#[test] +fn independently_qualified_regular_alternative_takes_priority_over_short_pinned_progress() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.timeline_fingerprint.segment_count = 1; + pinned.timeline_fingerprint.segment_samples.truncate(1); + let alternatives = [ + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(2, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + assert_eq!( + evaluate( + &[pinned.clone(), alternatives[0].clone(), alternatives[1].clone()], + HlsManifestAcceptanceTrigger::RecoveryRequired + ), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch + } + ); + let mut unstageable = alternatives[0].clone(); + unstageable.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; + assert_eq!( + evaluate(&[pinned, unstageable], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); +} diff --git a/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs b/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs new file mode 100644 index 000000000..337c88712 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/tests/reduced_retry.rs @@ -0,0 +1,190 @@ +use super::*; +use crate::manifest_acceptance::episode::HlsManifestAcceptanceEpisodeOutcome; +use shared::model::HlsManifestRecoveryBurstLevel; + +fn selectable_episode() -> HlsManifestAcceptanceEpisode { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(7), + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &episode_timing(100, plan), + ); + episode.record_full_burst(); + episode.state = HlsManifestAcceptanceState::Committing; + episode.observed_landscape = + Some(manifest_acceptance_landscape(&[observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost)])); + episode +} + +fn identity(marker: usize) -> HlsManifestRecoveryCandidateIdentity { + HlsManifestRecoveryCandidateIdentity::from_candidate(marker, Some("origin-a"), "manifest") +} + +fn assert_rejected_unchanged( + episode: &mut HlsManifestAcceptanceEpisode, + generation: HlsManifestAcceptanceGeneration, + candidate: HlsManifestRecoveryCandidateIdentity, + expected: HlsRecoveryWorkloadBindingUpdate, +) { + let before = episode.clone(); + assert_eq!(episode.select_candidate_for_reduced_retry(generation, candidate), expected); + assert_eq!(*episode, before, "a rejected selection is atomic"); +} + +#[test] +fn reduced_retry_reopens_only_eligible_exhaustion_and_preserves_frozen_evidence() { + for outcome in [ + HlsManifestAcceptanceEpisodeOutcome::Pending, + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(HlsManifestAcceptanceExhaustionReason::AllFailed), + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(HlsManifestAcceptanceExhaustionReason::NoProgress), + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( + HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, + ), + ] { + let mut episode = selectable_episode(); + episode.outcome = outcome; + let generation = episode.generation; + let before = episode.clone(); + if outcome != HlsManifestAcceptanceEpisodeOutcome::Pending { + assert_eq!( + episode.select_candidate(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::EpisodeInactive + ); + assert_eq!(episode, before, "normal selection still rejects exhaustion"); + } + let mut expected = before; + expected.outcome = HlsManifestAcceptanceEpisodeOutcome::Pending; + assert_eq!(expected.select_candidate(generation, identity(0)), HlsRecoveryWorkloadBindingUpdate::Applied); + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!(episode, expected, "only outcome and the selected identity may change"); + } +} + +#[test] +fn reduced_retry_rejects_stale_generation_and_inactive_state_without_mutation() { + let mut episode = selectable_episode(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoProgress); + let generation = episode.generation; + assert_rejected_unchanged( + &mut episode, + HlsManifestAcceptanceGeneration(generation.0 + 1), + identity(0), + HlsRecoveryWorkloadBindingUpdate::StaleGeneration, + ); + for state in [ + HlsManifestAcceptanceState::Holding, + HlsManifestAcceptanceState::Collecting, + HlsManifestAcceptanceState::Completed, + ] { + episode.state = state; + assert_rejected_unchanged( + &mut episode, + generation, + identity(0), + HlsRecoveryWorkloadBindingUpdate::EpisodeInactive, + ); + } +} + +#[test] +fn reduced_retry_keeps_identity_and_bound_workload_guards_until_holding_reset() { + let mut episode = selectable_episode(); + let generation = episode.generation; + assert_eq!(episode.select_candidate(generation, identity(0)), HlsRecoveryWorkloadBindingUpdate::Applied); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); + assert_rejected_unchanged( + &mut episode, + generation, + identity(1), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(0)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + let workload = HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstCompleted, + segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, + map: HlsRecoveryMapWorkload::NotRequired, + }; + assert_eq!( + episode.bind_selected_candidate(generation, identity(0), workload), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); + assert_rejected_unchanged( + &mut episode, + generation, + identity(0), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + assert_rejected_unchanged( + &mut episode, + generation, + identity(1), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch, + ); + + episode.hold_after_uncommitted_burst(None, Some(500)); + episode.state = HlsManifestAcceptanceState::Committing; + assert_eq!( + episode.select_candidate_for_reduced_retry(generation, identity(1)), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!(episode.selected_candidate_identity(), Some(identity(1))); +} + +#[test] +fn reduced_retry_never_reopens_incomplete_committed_or_deterministic_episodes() { + let mut episode = selectable_episode(); + let generation = episode.generation; + episode.full_burst_completed = false; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + episode.full_burst_completed = true; + episode.complete(); + // Even an otherwise selectable state cannot reopen a committed outcome. + episode.state = HlsManifestAcceptanceState::Committing; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + + let resource_key = HlsMediaResourceIdentity::for_test(7).semantic_key(); + let conflict = HlsDeterministicTimelineConflict { + previous_proxy_tail: Some(2), + existing_proxy_seq: 0, + candidate_position: 1, + candidate_origin_seq: 491, + resource_key, + decision: crate::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, + candidate_fingerprint: crate::deterministic_conflict::HlsDeterministicConflictFingerprint { + segment_count: 1, + first_program_date_time_ms: None, + last_program_date_time_ms: None, + duration_pattern_hash: [1; 32], + discontinuity_pattern_hash: [2; 32], + semantic_resource_pattern_hash: Some(resource_key.bytes()), + map_and_encryption_hash: [3; 32], + container_signature_hash: [4; 32], + segment_samples: Vec::new(), + }, + }; + episode = selectable_episode(); + episode.record_deterministic_conflict(HlsDeterministicConflictReceipt { + conflict, + origin_progress_generation: 1, + published_resource_history_generation: 1, + pinned_host_generation: 1, + }); + episode.state = HlsManifestAcceptanceState::Committing; + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + // A receipt remains authoritative independently of how another caller recorded exhaustion. + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoProgress); + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); + episode.deterministic_conflict_receipt = None; + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict); + assert_rejected_unchanged(&mut episode, generation, identity(0), HlsRecoveryWorkloadBindingUpdate::EpisodeInactive); +} diff --git a/backend/hls/src/manifest_commit.rs b/backend/hls/src/manifest_commit.rs index c454e4dd2..efa0c8d7f 100644 --- a/backend/hls/src/manifest_commit.rs +++ b/backend/hls/src/manifest_commit.rs @@ -183,6 +183,10 @@ fn origin_window_is_fresh(session: &HlsSession, now_ms: u64) -> bool { .is_some_and(|confirmed_at_ms| now_ms.saturating_sub(confirmed_at_ms) <= max_age_ms) } +pub(crate) fn has_servable_committed_manifest(session: &HlsSession, now_ms: u64) -> bool { + hls_committed_manifest_for_request(session, now_ms).is_some() +} + fn hls_committed_manifest_for_request(session: &HlsSession, now_ms: u64) -> Option { let protection = session.account_binding_protection(now_ms); hls_committed_manifest_candidate(session).filter(|candidate| { diff --git a/backend/hls/src/manifest_fetch/episode.rs b/backend/hls/src/manifest_fetch/episode.rs index 04c2dce07..41b6189a1 100644 --- a/backend/hls/src/manifest_fetch/episode.rs +++ b/backend/hls/src/manifest_fetch/episode.rs @@ -8,7 +8,10 @@ use super::{ current_time_millis, fingerprint::build_manifest_timeline_fingerprint, - recovery::{acceptance_attempt_may_start, HlsManifestRecoveryAttemptPlan, HlsManifestRecoveryCandidate}, + recovery::{ + acceptance_attempt_may_start, HlsManifestRecoveryAttemptPlan, HlsManifestRecoveryCandidate, + HlsManifestRequalificationWindow, + }, HlsManifestOriginRelation, HlsManifestSequenceRelation, HlsOriginManifestFetchContext, HLS_COMMITTED_CONTENT_ANCHOR_PROBE_LIMIT, }; @@ -23,32 +26,53 @@ use crate::{ HlsSwitchSegmentReadiness, }, recovery_timing::{ - HlsAcceptanceDeadlineMs, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, - HlsAcceptanceEpisodeTimingSeed, HlsRecoveryWorkloadEnvelope, HlsTerminalMediaPreparationState, - HlsTransitionMarginMs, + HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsAcceptanceEpisodeTimingSeed, + HlsRecoveryWorkloadEnvelope, HlsTerminalMediaPreparationState, HlsTransitionMarginMs, }, resource_identity::HlsMediaResourceIdentity, }; use log::debug; use shared::model::HlsManifestRecoveryBurstPlan; +pub(super) fn manifest_acceptance_generation_is_current( + session: &crate::HlsSession, + generation: HlsManifestAcceptanceGeneration, +) -> bool { + session.origin_control.acceptance_generation == generation + && session.origin_control.acceptance_episode.as_ref().is_some_and(|episode| episode.generation == generation) +} + pub(super) async fn begin_manifest_acceptance_episode( context: &HlsOriginManifestFetchContext, plan: &HlsManifestRecoveryAttemptPlan<'_>, burst_plan: HlsManifestRecoveryBurstPlan, + expected_generation: Option, ) -> Option { let mut session = context.session.write().await; + // Keep ownership validation atomic with the following episode/qualification changes. + if expected_generation.is_some_and(|generation| !manifest_acceptance_generation_is_current(&session, generation)) { + return None; + } + let basis = crate::short_manifest::HlsOriginQualificationBasis::capture(&session); + if !session.origin_control.qualification.pending_is_current(&basis) { + return None; + } if plan.attempt_index != 0 { let episode = session.origin_control.acceptance_episode.as_mut()?; if episode.trigger() != plan.trigger || episode.state == HlsManifestAcceptanceState::Completed { return None; } episode.state = HlsManifestAcceptanceState::Collecting; - return Some(episode.generation); + let generation = episode.generation; + session.origin_control.qualification = + crate::short_manifest::HlsOriginQualification::Checking { basis, generation, target: plan.binding.clone() }; + return Some(generation); } let started_at_ms = current_time_millis(); let timing = acceptance_episode_timing(context, &session, started_at_ms, burst_plan); let generation = session.origin_control.begin_acceptance_episode(started_at_ms, burst_plan, plan.trigger, &timing); + session.origin_control.qualification = + crate::short_manifest::HlsOriginQualification::Checking { basis, generation, target: plan.binding.clone() }; if let Some(episode) = session.origin_control.acceptance_episode.as_mut() { episode.state = HlsManifestAcceptanceState::Collecting; debug!( @@ -145,36 +169,51 @@ pub(super) async fn begin_requalified_manifest_acceptance_episode( context: &HlsOriginManifestFetchContext, generation: Option, trigger: HlsManifestAcceptanceTrigger, - acceptance_deadline: HlsAcceptanceDeadlineMs, -) -> bool { - let Some(generation) = generation else { - return false; - }; + window: &HlsManifestRequalificationWindow, +) -> Option { + let generation = generation?; let mut session = context.session.write().await; let now_ms = current_time_millis(); - if !acceptance_attempt_may_start(false, now_ms, 0, acceptance_deadline) { - return false; + match window { + HlsManifestRequalificationWindow::CurrentEpisode(deadline) + if !acceptance_attempt_may_start(false, now_ms, 0, *deadline) => + { + return None + } + HlsManifestRequalificationWindow::CurrentEpisode(_) + | HlsManifestRequalificationWindow::ResumedAfterSharedBackoff => {} } - let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { - return false; - }; + let basis = crate::short_manifest::HlsOriginQualificationBasis::capture(&session); + if !matches!( + &session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Checking { basis: current, generation: current_generation, .. } + if current == &basis && *current_generation == generation + ) { + return None; + } + let episode = session.origin_control.acceptance_episode.as_mut()?; if episode.generation != generation || !episode.full_burst_completed || episode.state == HlsManifestAcceptanceState::Completed { - return false; + return None; } episode.state = HlsManifestAcceptanceState::Holding; let configured_plan = context.manifest_recovery_burst.level.plan(); let timing = acceptance_episode_timing(context, &session, now_ms, configured_plan); let next_generation = session.origin_control.begin_acceptance_episode(now_ms, configured_plan, trigger, &timing); + if let crate::short_manifest::HlsOriginQualification::Checking { generation, .. } = + &mut session.origin_control.qualification + { + *generation = next_generation; + } debug!( "HLS manifest acceptance landscape changed: previous_generation={} next_generation={} candidates={} decision=full-requalification", generation.0, next_generation.0, configured_plan.total_candidates() ); - true + Some(next_generation) } pub(super) async fn record_full_manifest_acceptance_burst( @@ -281,7 +320,10 @@ pub(super) async fn record_manifest_acceptance_exhaustion( let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { return; }; - if episode.generation == generation { + if episode.generation == generation + && episode.exhaustion_reason().is_none() + && episode.deterministic_conflict_receipt().is_none() + { episode.record_exhaustion(reason); } } diff --git a/backend/hls/src/manifest_fetch/fetch.rs b/backend/hls/src/manifest_fetch/fetch.rs index 06c7c1640..a9e3e732e 100644 --- a/backend/hls/src/manifest_fetch/fetch.rs +++ b/backend/hls/src/manifest_fetch/fetch.rs @@ -5,15 +5,15 @@ use super::{ origin_manifest_fetch_error_from_io_error, resolved_hls_manifest_request_url_from_input, response_to_fetched_manifest, same_origin, ManifestRecoveryAttemptLogContext, }, - model::{FetchedOriginManifest, HlsOriginManifestFetchContext, RetryPolicy}, + model::{FetchedOriginManifest, HlsOriginManifestFetchContext, RetryPolicy, DEFAULT_HLS_TARGET_DURATION_SECS}, selection_log::{log_manifest_initial_attempt, log_manifest_retry_scheduled, ManifestRetryLogKind}, }; #[cfg(any(test, feature = "test-support"))] use super::{model::LiveHlsOriginEntry, selection_log::log_origin_refresh_retry_scheduled}; use crate::{ - append_hls_provider_session_headers, headers::merge_hls_provider_session_headers, - manifest_origin_binding::HlsManifestOriginBinding, observability::hls_origin_log_value, scrub_hls_origin_headers, - MAX_MANUAL_REDIRECTS, + append_hls_provider_session_headers, availability::record_retryable_manifest_attempt_failure, + headers::merge_hls_provider_session_headers, manifest_origin_binding::HlsManifestOriginBinding, + observability::hls_origin_log_value, scrub_hls_origin_headers, MAX_MANUAL_REDIRECTS, }; use axum::http::HeaderMap; use log::debug; @@ -82,13 +82,10 @@ async fn fetch_hls_origin_manifest_initial_global_policy( ) -> Result { log_manifest_initial_attempt(context).await; let input_source = context.origin_entry.to_input_source(); - let account_binding = { + let (account_binding, progress_generation) = { let session = context.session.read().await; - if session.origin_account_binding.is_some() { - "present" - } else { - "absent" - } + let account_binding = if session.origin_account_binding.is_some() { "present" } else { "absent" }; + (account_binding, session.origin_control.progress_generation) }; debug!( "HLS origin manifest request started: account_binding={account_binding} request_url={}", @@ -151,8 +148,13 @@ async fn fetch_hls_origin_manifest_initial_global_policy( match fetch_result { Ok(fetched) => return Ok(fetched.with_attempts(attempt_index + 1)), Err(err) if is_hls_retryable_initial_manifest_fetch_error(&err) && attempt_index + 1 < attempts => { + record_retryable_manifest_attempt_failure(context, progress_generation, &err, current_time_millis()) + .await; let retry_after_ms = match &err { - OriginManifestFetchError::RetryableStatus(_, retry_after_ms) => *retry_after_ms, + OriginManifestFetchError::RetryableStatus(_, Some(retry_after_ms)) => { + let session = context.session.read().await; + Some(cap_manifest_retry_after_ms(&session, *retry_after_ms)) + } _ => None, }; let jitter_ms = if retry_after_ms.is_some() { 0 } else { context.retry_policy.sample_jitter_ms() }; @@ -284,6 +286,16 @@ pub(super) fn single_variant_url(body: &str, master_url: &str) -> Option { matches!(url.scheme(), "http" | "https").then_some(url) } +pub(super) fn cap_manifest_retry_after_ms(session: &crate::HlsSession, retry_after_ms: u64) -> u64 { + let target_duration_ms = session + .origin_control + .target_duration_snapshot_ms + .filter(|duration_ms| *duration_ms > 0) + .or_else(|| session.target_duration.filter(|seconds| *seconds > 0).map(|seconds| u64::from(seconds) * 1_000)) + .unwrap_or(u64::from(DEFAULT_HLS_TARGET_DURATION_SECS) * 1_000); + retry_after_ms.min(target_duration_ms / 2) +} + pub(super) fn next_retry_delay_ms( retry_policy: &RetryPolicy, attempt_index: usize, diff --git a/backend/hls/src/manifest_fetch/model.rs b/backend/hls/src/manifest_fetch/model.rs index bd22705c0..e2e913d8c 100644 --- a/backend/hls/src/manifest_fetch/model.rs +++ b/backend/hls/src/manifest_fetch/model.rs @@ -240,6 +240,8 @@ impl FetchedOriginManifest { pub struct HlsOriginManifestFetchContext { pub app_config: Arc, pub session: HlsSessionHandle, + /// Availability owner used to react to failed attempts while the logical fetch is still in flight. + pub availability_runtime: Option, pub origin_entry: LiveHlsOriginEntry, pub headers: HeaderMap, pub client: Client, diff --git a/backend/hls/src/manifest_fetch/quality.rs b/backend/hls/src/manifest_fetch/quality.rs index 0c1974f0b..28422b32b 100644 --- a/backend/hls/src/manifest_fetch/quality.rs +++ b/backend/hls/src/manifest_fetch/quality.rs @@ -228,6 +228,9 @@ fn classify_manifest_sequence_relation( } fn manifest_continuity_mode(session: &crate::HlsSession, now_ms: u64) -> HlsManifestContinuityMode { + if session.origin_is_qualified() { + return HlsManifestContinuityMode::StrictContinuity; + } if session.origin_seq_highwater.is_none() { return HlsManifestContinuityMode::RebaseAllowed; } diff --git a/backend/hls/src/manifest_fetch/recovery.rs b/backend/hls/src/manifest_fetch/recovery.rs index bf37650b3..8f79e904c 100644 --- a/backend/hls/src/manifest_fetch/recovery.rs +++ b/backend/hls/src/manifest_fetch/recovery.rs @@ -10,7 +10,8 @@ use super::{ episode::{ begin_manifest_acceptance_episode, begin_requalified_manifest_acceptance_episode, committed_resource_evidence, complete_manifest_acceptance_episode, hold_uncommitted_manifest_acceptance_episode, - manifest_acceptance_episode_snapshot, manifest_acceptance_exhaustion_reason, manifest_candidate_observation, + manifest_acceptance_episode_snapshot, manifest_acceptance_exhaustion_reason, + manifest_acceptance_generation_is_current, manifest_candidate_observation, record_full_manifest_acceptance_burst, record_manifest_acceptance_exhaustion, record_manifest_acceptance_landscape, update_manifest_acceptance_episode_state, }, @@ -46,6 +47,7 @@ use crate::{ manifest_origin_binding::HlsManifestOriginBinding, recovery_timing::HlsAcceptanceDeadlineMs, }; +use log::debug; use shared::model::{HlsManifestRecoveryBurstLevel, HlsManifestRecoveryBurstPlan}; use std::{ future::Future, @@ -56,7 +58,7 @@ use tokio::task::JoinSet; enum HlsManifestRecoveryAttemptError { Fetch(OriginManifestFetchError), Rejected(HlsManifestRejectLogReason), - Requalified, + Requalified(HlsManifestAcceptanceGeneration), Committed(T), } @@ -93,10 +95,15 @@ where } let attempts = context.retry_policy.attempt_count(); let mut last_error = OriginManifestFetchError::RetryExhausted; - let mut next_attempt_is_full_plan = true; + let (resume, mut expected_generation) = { + let session = context.session.read().await; + let resume = session.origin_qualification_retry_target().is_some(); + (resume, resume.then_some(session.origin_control.acceptance_generation)) + }; + let mut next_attempt_is_full_plan = !resume; let mut requalifications = 0_u8; - let mut attempt_index = 0_usize; - let mut attempt_limit = attempts; + let mut attempt_index = usize::from(resume); + let mut attempt_limit = if resume { 2 } else { attempts }; let mut completed_candidate_requests = 0_usize; while attempt_index < attempt_limit { // A materially changed landscape is requalified immediately. Once @@ -108,13 +115,15 @@ where let jitter = context.retry_policy.sample_jitter_ms(); context.retry_policy.delay_for_attempt_ms(attempt_index, jitter).unwrap_or_default() }; - let acceptance_deadline = current_acceptance_deadline(context).await; - if !acceptance_attempt_may_start( - next_attempt_is_full_plan, - current_time_millis(), - delay_ms, - acceptance_deadline, - ) { + let acceptance_deadline = current_acceptance_deadline(context, expected_generation).await?; + if !resume + && !acceptance_attempt_may_start( + next_attempt_is_full_plan, + current_time_millis(), + delay_ms, + acceptance_deadline, + ) + { return Err(last_error); } if delay_ms > 0 { @@ -131,30 +140,29 @@ where trigger, current_burst_is_full_plan: next_attempt_is_full_plan, may_requalify: requalifications < HLS_MANIFEST_ACCEPTANCE_MAX_REQUALIFICATIONS_PER_REFRESH, - acceptance_deadline, + requalification_window: if resume && attempt_index == 1 { + HlsManifestRequalificationWindow::ResumedAfterSharedBackoff + } else { + HlsManifestRequalificationWindow::CurrentEpisode(acceptance_deadline) + }, completed_candidate_requests, }; let candidate_requests_in_attempt = recovery_burst_plan(context, next_attempt_is_full_plan).total_candidates(); next_attempt_is_full_plan = false; - match fetch_and_commit_manifest_recovery_attempt(context, attempt_plan, &mut commit).await { + match fetch_and_commit_manifest_recovery_attempt(context, attempt_plan, &mut expected_generation, &mut commit) + .await + { HlsManifestRecoveryAttemptError::Committed(committed) => return Ok(committed), - HlsManifestRecoveryAttemptError::Requalified => { + HlsManifestRecoveryAttemptError::Requalified(generation) => { + // Only an explicitly authorized requalification may transfer chain ownership. + expected_generation = Some(generation); requalifications = requalifications.saturating_add(1); next_attempt_is_full_plan = true; last_error = OriginManifestFetchError::RetryExhausted; attempt_limit = attempt_limit_for_started_requalification(attempt_limit, attempt_index); } HlsManifestRecoveryAttemptError::Rejected(reason) if attempt_index.saturating_add(1) < attempt_limit => { - log_manifest_retry_scheduled( - context, - ManifestRetryLogKind::PinnedHostRecovery, - attempt_index, - attempt_limit, - next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), - Some(&reason), - None, - ) - .await; + log_manifest_recovery_retry(context, attempt_index, attempt_limit, Some(&reason), None).await; reject_reason = Some(reason); last_error = OriginManifestFetchError::RetryExhausted; } @@ -165,16 +173,7 @@ where if is_hls_retryable_manifest_reject_fetch_error(&err) && attempt_index.saturating_add(1) < attempt_limit => { - log_manifest_retry_scheduled( - context, - ManifestRetryLogKind::PinnedHostRecovery, - attempt_index, - attempt_limit, - next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), - None, - Some(&err), - ) - .await; + log_manifest_recovery_retry(context, attempt_index, attempt_limit, None, Some(&err)).await; last_error = err; } HlsManifestRecoveryAttemptError::Fetch(err) => return Err(err), @@ -186,6 +185,25 @@ where Err(last_error) } +async fn log_manifest_recovery_retry( + context: &HlsOriginManifestFetchContext, + attempt_index: usize, + attempt_limit: usize, + reject_reason: Option<&HlsManifestRejectLogReason>, + fetch_error: Option<&OriginManifestFetchError>, +) { + log_manifest_retry_scheduled( + context, + ManifestRetryLogKind::PinnedHostRecovery, + attempt_index, + attempt_limit, + next_retry_delay_ms(&context.retry_policy, attempt_index, None, 0), + reject_reason, + fetch_error, + ) + .await; +} + pub(super) fn acceptance_attempt_may_start( current_burst_is_full_plan: bool, now_ms: u64, @@ -198,11 +216,20 @@ pub(super) fn acceptance_attempt_may_start( current_burst_is_full_plan || now_ms.saturating_add(delay_ms) < acceptance_deadline.as_millis_since_epoch() } -async fn current_acceptance_deadline(context: &HlsOriginManifestFetchContext) -> HlsAcceptanceDeadlineMs { - context.session.read().await.origin_control.acceptance_episode.as_ref().map_or_else( +async fn current_acceptance_deadline( + context: &HlsOriginManifestFetchContext, + expected_generation: Option, +) -> Result { + let session = context.session.read().await; + if expected_generation.is_some_and(|generation| !manifest_acceptance_generation_is_current(&session, generation)) { + return Err(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + Ok(session.origin_control.acceptance_episode.as_ref().map_or_else( || HlsAcceptanceDeadlineMs::from_millis_since_epoch(u64::MAX), |episode| episode.timing().acceptance_deadline, - ) + )) } pub(super) fn attempt_limit_for_started_requalification(attempt_limit: usize, attempt_index: usize) -> usize { @@ -212,6 +239,12 @@ pub(super) fn attempt_limit_for_started_requalification(attempt_limit: usize, at attempt_limit.max(attempt_index.saturating_add(2)) } +/// A due session retry can open a fresh episode only after changed origin evidence. +pub(super) enum HlsManifestRequalificationWindow { + CurrentEpisode(HlsAcceptanceDeadlineMs), + ResumedAfterSharedBackoff, +} + pub(super) struct HlsManifestRecoveryAttemptPlan<'a> { pub(super) binding: &'a HlsManifestOriginBinding, pub(super) attempt_index: usize, @@ -222,13 +255,14 @@ pub(super) struct HlsManifestRecoveryAttemptPlan<'a> { pub(super) trigger: HlsManifestAcceptanceTrigger, current_burst_is_full_plan: bool, may_requalify: bool, - acceptance_deadline: HlsAcceptanceDeadlineMs, + requalification_window: HlsManifestRequalificationWindow, completed_candidate_requests: usize, } async fn fetch_and_commit_manifest_recovery_attempt( context: &HlsOriginManifestFetchContext, plan: HlsManifestRecoveryAttemptPlan<'_>, + expected_generation: &mut Option, commit: &mut C, ) -> HlsManifestRecoveryAttemptError where @@ -236,20 +270,27 @@ where Fut: Future>, { let burst_plan = recovery_burst_plan(context, plan.current_burst_is_full_plan); - fetch_and_commit_manifest_recovery_burst_attempt(context, plan, burst_plan, commit).await + fetch_and_commit_manifest_recovery_burst_attempt(context, plan, burst_plan, expected_generation, commit).await } async fn fetch_and_commit_manifest_recovery_burst_attempt( context: &HlsOriginManifestFetchContext, plan: HlsManifestRecoveryAttemptPlan<'_>, burst_plan: HlsManifestRecoveryBurstPlan, + expected_generation: &mut Option, commit: &mut C, ) -> HlsManifestRecoveryAttemptError where C: FnMut(FetchedOriginManifest, HlsManifestCommitAcceptanceMode) -> Fut, Fut: Future>, { - let episode_generation = begin_manifest_acceptance_episode(context, &plan, burst_plan).await; + let episode_generation = begin_manifest_acceptance_episode(context, &plan, burst_plan, *expected_generation).await; + if episode_generation.is_none() { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + *expected_generation = episode_generation; let HlsManifestRecoveryBurstCollection { fetched_candidates, completed_candidates, @@ -273,14 +314,19 @@ where OriginManifestFetchError::DeterministicTimelineConflict(conflict), ); } - HlsManifestRecoveryBurstEvaluationOutcome::Requalified => { - return HlsManifestRecoveryAttemptError::Requalified; + HlsManifestRecoveryBurstEvaluationOutcome::Requalified(generation) => { + return HlsManifestRecoveryAttemptError::Requalified(generation); } HlsManifestRecoveryBurstEvaluationOutcome::Exhausted => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RetryExhausted); } + HlsManifestRecoveryBurstEvaluationOutcome::Superseded => { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } }; - match commit_selected_manifest_recovery_candidate( + let selection_resumed = match commit_selected_manifest_recovery_candidate( context, &plan, burst_plan, @@ -295,7 +341,19 @@ where HlsSelectedManifestRecoveryCommit::Committed(committed) => { return HlsManifestRecoveryAttemptError::Committed(committed); } - HlsSelectedManifestRecoveryCommit::Rejected(reason) => last_reject_reason = Some(reason), + HlsSelectedManifestRecoveryCommit::Rejected(reason) => { + last_reject_reason = Some(reason); + true + } + HlsSelectedManifestRecoveryCommit::SelectionRejected(HlsRecoveryWorkloadBindingUpdate::StaleGeneration) => { + return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::RecoveryUnavailable { + reason: super::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }); + } + HlsSelectedManifestRecoveryCommit::SelectionRejected(_) => { + last_reject_reason = Some(HlsManifestRejectLogReason::StagedSwitchInvalidated); + false + } HlsSelectedManifestRecoveryCommit::LocalRepresentationLimit(violation) => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::LocalRepresentationLimit( violation, @@ -307,22 +365,16 @@ where HlsSelectedManifestRecoveryCommit::CommitGenerationExhausted => { return HlsManifestRecoveryAttemptError::Fetch(OriginManifestFetchError::CommitGenerationExhausted); } - HlsSelectedManifestRecoveryCommit::NotSelected => {} - } + HlsSelectedManifestRecoveryCommit::NotSelected => false, + }; - if plan.current_burst_is_full_plan { - record_manifest_acceptance_exhaustion( - context, - episode_generation, - manifest_acceptance_exhaustion_reason(&evaluation.observations), - ) - .await; - } - hold_uncommitted_manifest_acceptance_episode( + hold_manifest_recovery_burst( context, episode_generation, + &evaluation.observations, evaluation.held_alternative, evaluation.next_retry_at_ms, + plan.current_burst_is_full_plan || selection_resumed, ) .await; if let Some(reason) = last_reject_reason { @@ -331,6 +383,25 @@ where HlsManifestRecoveryAttemptError::Fetch(last_fetch_error.unwrap_or(OriginManifestFetchError::RetryExhausted)) } +async fn hold_manifest_recovery_burst( + context: &HlsOriginManifestFetchContext, + episode_generation: Option, + observations: &[HlsManifestCandidateObservation], + held_alternative: Option, + next_retry_at_ms: u64, + record_exhaustion: bool, +) { + if record_exhaustion { + record_manifest_acceptance_exhaustion( + context, + episode_generation, + manifest_acceptance_exhaustion_reason(observations), + ) + .await; + } + hold_uncommitted_manifest_acceptance_episode(context, episode_generation, held_alternative, next_retry_at_ms).await; +} + struct HlsManifestRecoveryBurstEvaluation { observations: Vec, acceptance_plan: HlsManifestCommitPlan, @@ -341,8 +412,9 @@ struct HlsManifestRecoveryBurstEvaluation { enum HlsManifestRecoveryBurstEvaluationOutcome { Continue(Box), DeterministicConflict(Box), - Requalified, + Requalified(HlsManifestAcceptanceGeneration), Exhausted, + Superseded, } const fn manifest_acceptance_state_for_plan(plan: &HlsManifestCommitPlan) -> HlsManifestAcceptanceState { @@ -396,6 +468,10 @@ async fn evaluate_manifest_recovery_burst( } } let episode_snapshot = manifest_acceptance_episode_snapshot(context, episode_generation).await; + if episode_snapshot.is_none() { + // An old response must not let the next attempt join a successor episode. + return HlsManifestRecoveryBurstEvaluationOutcome::Superseded; + } let reduced_landscape_changed = !plan.current_burst_is_full_plan && episode_snapshot .as_ref() @@ -403,16 +479,17 @@ async fn evaluate_manifest_recovery_burst( .map(|landscape| classify_reduced_retry_landscape(landscape, &observations)) .is_some_and(HlsReducedRetryLandscapeChange::requires_full_requalification); if reduced_landscape_changed { - if plan.may_requalify - && begin_requalified_manifest_acceptance_episode( + if plan.may_requalify { + if let Some(generation) = begin_requalified_manifest_acceptance_episode( context, episode_generation, plan.trigger, - plan.acceptance_deadline, + &plan.requalification_window, ) .await - { - return HlsManifestRecoveryBurstEvaluationOutcome::Requalified; + { + return HlsManifestRecoveryBurstEvaluationOutcome::Requalified(generation); + } } // A changed landscape may never fall through into reduced-burst // cross-host acceptance when its full requalification budget is gone. @@ -458,6 +535,7 @@ async fn evaluate_manifest_recovery_burst( enum HlsSelectedManifestRecoveryCommit { Committed(T), Rejected(HlsManifestRejectLogReason), + SelectionRejected(HlsRecoveryWorkloadBindingUpdate), LocalRepresentationLimit(crate::manifest_limits::HlsManifestLimitViolation), MalformedTransientRepresentation, CommitGenerationExhausted, @@ -493,8 +571,15 @@ where report.quality.effective_host.as_deref(), &fetched.body, ); - if !select_manifest_recovery_candidate(context, episode_generation, candidate_identity).await { - return HlsSelectedManifestRecoveryCommit::Rejected(HlsManifestRejectLogReason::StagedSwitchInvalidated); + let selection_update = select_manifest_recovery_candidate( + context, + episode_generation, + candidate_identity, + plan.current_burst_is_full_plan, + ) + .await; + if selection_update != HlsRecoveryWorkloadBindingUpdate::Applied { + return HlsSelectedManifestRecoveryCommit::SelectionRejected(selection_update); } let acceptance_mode = selected_commit_acceptance_mode(selected_commit_kind, plan.acceptance_mode); let selection = if burst_plan.total_candidates() > 1 { @@ -558,15 +643,36 @@ async fn select_manifest_recovery_candidate( context: &HlsOriginManifestFetchContext, generation: Option, candidate_identity: HlsManifestRecoveryCandidateIdentity, -) -> bool { + current_burst_is_full_plan: bool, +) -> HlsRecoveryWorkloadBindingUpdate { + let attempt_type = if current_burst_is_full_plan { "full" } else { "reduced" }; let Some(generation) = generation else { - return false; + let session = context.session.read().await; + debug!("HLS manifest candidate selection rejected: update=EpisodeInactive attempt_type={attempt_type} expected_generation=none current_generation={} episode_status={:?}", + session.origin_control.acceptance_generation.0, + crate::manifest_acceptance::manifest_acceptance_episode_status(session.origin_control.acceptance_episode.as_ref(), session.origin_control.acceptance_generation, current_time_millis())); + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; }; let mut session = context.session.write().await; + let current_generation = session.origin_control.acceptance_generation; let Some(episode) = session.origin_control.acceptance_episode.as_mut() else { - return false; + debug!("HLS manifest candidate selection rejected: update=StaleGeneration attempt_type={attempt_type} expected_generation={} current_generation={} episode_status=Missing", generation.0, current_generation.0); + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; }; - episode.select_candidate(generation, candidate_identity) == HlsRecoveryWorkloadBindingUpdate::Applied + let update = if current_burst_is_full_plan { + episode.select_candidate(generation, candidate_identity) + } else { + episode.select_candidate_for_reduced_retry(generation, candidate_identity) + }; + if update != HlsRecoveryWorkloadBindingUpdate::Applied { + debug!( + "HLS manifest candidate selection rejected: update={update:?} attempt_type={attempt_type} expected_generation={} current_generation={} episode_status={:?}", + generation.0, + episode.generation.0, + crate::manifest_acceptance::manifest_acceptance_episode_status(Some(episode), current_generation, current_time_millis()), + ); + } + update } const fn selected_commit_acceptance_mode( diff --git a/backend/hls/src/manifest_fetch/tests.rs b/backend/hls/src/manifest_fetch/tests.rs index 69791fe37..e25f6731d 100644 --- a/backend/hls/src/manifest_fetch/tests.rs +++ b/backend/hls/src/manifest_fetch/tests.rs @@ -8,6 +8,7 @@ use super::{ episode::candidate_resource_timeline_evidence, error::{HlsManifestRejectLogReason, OriginManifestFetchError}, + fetch::cap_manifest_retry_after_ms, fingerprint::{ build_manifest_timeline_fingerprint, deterministic_conflict_fingerprint, deterministic_timeline_conflict_from_rejection, @@ -28,6 +29,7 @@ use crate::{ recovery_timing::HlsAcceptanceDeadlineMs, resource_identity::{HlsMediaResourceIdentity, HlsMediaResourceSemanticKey}, timeline::HlsResourceReplayDecision, + HlsSession, HlsSessionKey, }; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use flate2::{ @@ -384,6 +386,38 @@ fn initial_manifest_retry_delay_uses_next_slot_bounded_jitter_and_retry_after_ov assert_eq!(super::next_retry_delay_ms(&retry_policy, 0, Some(3_000), 25), 3_000); } +#[test] +fn manifest_retry_after_is_capped_at_half_the_origin_target_duration() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_control.record_media_progress(50, 12_000); + session.target_duration = Some(2); + + assert_eq!( + [0, 3_000, 6_000, 60_000, u64::MAX].map(|delay_ms| cap_manifest_retry_after_ms(&session, delay_ms)), + [0, 3_000, 6_000, 6_000, 6_000] + ); +} + +#[test] +fn manifest_retry_after_uses_session_target_duration_when_origin_timing_is_missing() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.target_duration = Some(3); + + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 1_500); + session.origin_control.target_duration_snapshot_ms = Some(0); + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 1_500); +} + +#[test] +fn manifest_retry_after_uses_default_target_duration_before_first_manifest() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 7_500); + session.target_duration = Some(0); + session.origin_control.target_duration_snapshot_ms = Some(0); + assert_eq!(cap_manifest_retry_after_ms(&session, 60_000), 7_500); +} + #[test] fn initial_manifest_retry_status_is_not_labeled_as_recovery() { let error = OriginManifestFetchError::Timeout; diff --git a/backend/hls/src/map_fetcher.rs b/backend/hls/src/map_fetcher.rs index 8aa790b1b..5f101c3eb 100644 --- a/backend/hls/src/map_fetcher.rs +++ b/backend/hls/src/map_fetcher.rs @@ -1,10 +1,13 @@ use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, finish_hls_origin_account_io, - hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, - HlsAccessLeaseStore, HlsBoundAccountAcquireErrorKind, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, - HlsOriginIoContext, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchError, - HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSessionHandle, - MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, + build_hls_origin_resource_headers, hls_object_body_deadline, + run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, HlsAccessLeaseStore, + HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchKind, HlsResourceFetchSource, + HlsSegmentCache, HlsSessionHandle, MapCacheKey, MapCacheStatus, OriginMapFetchRef, ProxyMapId, SegmentFetchPolicy, +}; +use crate::origin_attempt::{ + prepare_media_origin_attempt, HlsMediaOriginAttemptGuard as MapOriginAttemptGuard, + HlsMediaOriginWorkFinish as MapOriginWorkFinish, }; use arc_swap::ArcSwap; use axum::http::HeaderMap; @@ -43,12 +46,6 @@ struct MapFetchCommit { generation_valid: bool, } -#[derive(Clone, Copy)] -struct MapOriginWorkFinish { - generation_valid: bool, - refresh_reservation: bool, -} - impl fmt::Debug for MapFetchSnapshot { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("MapFetchSnapshot") @@ -148,11 +145,18 @@ impl HlsMapWorkerPool { now_ms: u64, policy: &SegmentFetchPolicy, ) -> Option { - let (proxy_session_id, gc_marked_for_removal) = { + let (proxy_session_id, gc_marked_for_removal, needs_origin_io) = { let session = context.session.read().await; - (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + ( + session.proxy_session_id.clone(), + session.is_gc_marked_for_removal(), + session.maps.values().any(|entry| { + matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }) + && entry.origin_fetch_ref.is_some() + }), + ) }; - if gc_marked_for_removal { + if gc_marked_for_removal || !needs_origin_io { return None; } if !self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms) { @@ -164,6 +168,12 @@ impl HlsMapWorkerPool { } return None; } + if crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err() + { + return None; + } let mut session = context.session.write().await; if session.is_gc_marked_for_removal() { return None; @@ -173,7 +183,9 @@ impl HlsMapWorkerPool { } let proxy_map_id = session.maps.iter().find_map(|(proxy_map_id, entry)| { - matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }).then_some(*proxy_map_id) + (matches!(entry.status, MapCacheStatus::Discovered | MapCacheStatus::Queued { .. }) + && entry.origin_fetch_ref.as_ref().is_some_and(|fetch_ref| fetch_ref.is_valid_at(now_ms))) + .then_some(*proxy_map_id) })?; let origin_work_generation = session.activity.origin_work_generation; @@ -254,6 +266,7 @@ impl HlsMapWorkerPool { } } else if entry.cache_key == snapshot.cache_key && matches!(entry.status, MapCacheStatus::Fetching { .. }) + && !fetch_succeeded { entry.status = MapCacheStatus::Discovered; } @@ -277,6 +290,12 @@ impl HlsMapWorkerPool { if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await { debug!("HLS stale map cache cleanup failed: resource=map/{} error={err}", snapshot.proxy_map_id_log); } + let mut session = context.session.write().await; + if let Some(entry) = session.maps.get_mut(&snapshot.proxy_map_id).filter(|entry| { + entry.cache_key == snapshot.cache_key && matches!(entry.status, MapCacheStatus::Fetching { .. }) + }) { + entry.status = MapCacheStatus::Discovered; + } } if let (Some(coordinator), Some(proxy_session_id)) = (self.availability_reevaluations.as_ref(), evidence_changed_for.as_ref()) @@ -284,7 +303,8 @@ impl HlsMapWorkerPool { coordinator.notify_session_evidence_changed(proxy_session_id); } drop(permit); - if generation_valid { + if generation_valid || context.session.read().await.origin_handoff_ready_after(snapshot.origin_work_generation) + { self.schedule_wake(context, finished_at_ms); } } @@ -309,95 +329,23 @@ async fn fetch_map_into_cache( fetch_map_with_retries_into_cache(context, snapshot, policy).await } -struct MapOriginAttemptGuard { - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -} - async fn prepare_map_origin_attempt( context: MapFetchContext, policy: SegmentFetchPolicy, + expected_generation: u64, ) -> Result { - let started_generation = start_map_origin_work(&context).await; - let binding = - if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; - let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { - if binding.is_detached() { - finish_map_origin_work(&context, started_generation).await; - touch_map_origin_account_binding(&context, false).await; - return Err(MapFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); - } - let guard = match begin_hls_origin_account_io_bounded( - origin_io, - &context.session, - binding, - hls_object_body_deadline(policy.origin_segment_timeout_ms), - ) - .await - { - Ok(guard) => guard, - Err(err) => { - finish_map_origin_work(&context, started_generation).await; - touch_map_origin_account_binding(&context, false).await; - return Err(MapFetchError::ProviderUnavailable(err)); - } - }; - Some((origin_io.clone(), guard)) - } else { - None - }; - Ok(MapOriginAttemptGuard { started_generation, provider_lease }) + prepare_media_origin_attempt( + context.origin_io.as_ref(), + &context.session, + expected_generation, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + .map_err(MapFetchError::ProviderUnavailable) } async fn finish_map_origin_attempt(context: MapFetchContext, guard: MapOriginAttemptGuard) -> MapOriginWorkFinish { - finish_map_origin_io(&context, guard.started_generation, guard.provider_lease).await -} - -async fn finish_map_origin_io( - context: &MapFetchContext, - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -) -> MapOriginWorkFinish { - let origin_work = finish_map_origin_work(context, started_generation).await; - if let Some((origin_io, guard)) = provider_lease { - finish_hls_origin_account_io( - &origin_io, - &context.session, - guard, - origin_work.generation_valid && origin_work.refresh_reservation, - ) - .await; - touch_map_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) - .await; - } - origin_work -} - -async fn start_map_origin_work(context: &MapFetchContext) -> Option { - context.origin_io.as_ref()?; - let mut session = context.session.write().await; - Some(session.start_origin_work()) -} - -async fn finish_map_origin_work(context: &MapFetchContext, started_generation: Option) -> MapOriginWorkFinish { - let Some(started_generation) = started_generation else { - return MapOriginWorkFinish { generation_valid: true, refresh_reservation: false }; - }; - let mut session = context.session.write().await; - let generation_valid = session.finish_origin_work(started_generation); - let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); - MapOriginWorkFinish { generation_valid, refresh_reservation } -} - -async fn touch_map_origin_account_binding(context: &MapFetchContext, reservation_refreshed: bool) { - let mut session = context.session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = current_time_millis(); - binding.last_origin_io_at_ms = Some(now_ms); - if reservation_refreshed { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } + guard.finish(&context.session).await } #[allow(clippy::too_many_lines)] @@ -406,11 +354,8 @@ async fn fetch_map_with_retries_into_cache( snapshot: &MapFetchSnapshot, policy: &SegmentFetchPolicy, ) -> Result { - let headers = build_map_origin_headers( - &context.headers, - &context.origin_provider_session_headers, - snapshot.fetch_ref.byte_range, - )?; + let provider_headers = context.session.read().await.origin_provider_session_headers.clone(); + let headers = build_map_origin_headers(&context.headers, &provider_headers, snapshot.fetch_ref.byte_range)?; let target = HlsOriginResourceFetchTarget { kind: HlsResourceFetchKind::Map, source: HlsResourceFetchSource::Normal, @@ -432,6 +377,7 @@ async fn fetch_map_with_retries_into_cache( let session = context.session.read().await; super::HlsLogIdentity::from_session(&session) }; + let expected_generation = snapshot.origin_work_generation; let context = context.clone(); let snapshot = snapshot.clone(); let policy_for_prepare = policy.clone(); @@ -445,7 +391,7 @@ async fn fetch_map_with_retries_into_cache( move |_attempt| { let context = prepare_context.clone(); let policy = policy_for_prepare.clone(); - async move { prepare_map_origin_attempt(context, policy).await }.boxed() + async move { prepare_map_origin_attempt(context, policy, expected_generation).await }.boxed() }, move |guard| { let context = cleanup_context.clone(); @@ -570,6 +516,78 @@ mod tests { assert!(!headers.contains_key(header::HOST)); } + #[tokio::test] + async fn account_handoff_during_map_prepare_blocks_old_origin_request() { + use crate::{origin_attempt::tests::bound_origin_io, HlsOriginAccountIoLease, HlsOriginResourceFetchError}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("origin address"); + let requests = Arc::new(AtomicUsize::new(0)); + let seen = Arc::clone(&requests); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("origin request"); + let mut request = [0_u8; 2048]; + let read = socket.read(&mut request).await.expect("request reads"); + assert!(read > 0, "origin receives a request"); + seen.fetch_add(1, Ordering::AcqRel); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nmap") + .await + .expect("response writes"); + }); + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "map-handoff"), b"secret", 0).await; + let (io, replacement) = bound_origin_io(&session).await; + { + let mut session = session.write().await; + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n", + &format!("http://{addr}/account-a/index.m3u8"), + )) + .expect("manifest maps"); + session.queue_map_fetch_candidates(1); + } + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let worker = HlsMapWorkerPool::new(policy.clone()); + grant_usable_map_access_lease(&worker, &session.read().await.proxy_session_id).await; + let context = MapFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: Some(io), + }; + let snapshot = worker.next_fetch_snapshot(&context, 2, &policy).await.expect("map snapshot"); + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let read_blocker = session.read().await; + let mut fetch = Box::pin(super::fetch_map_with_retries_into_cache(&context, &snapshot, &policy)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + let mut handoff = Box::pin(session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + { + let mut session = handoff.await; + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + let result = fetch.await; + server.abort(); + assert_eq!(requests.load(Ordering::Acquire), 0, "stale MAP URL must never reach the origin"); + assert!(matches!(result, Err(HlsOriginResourceFetchError::ProviderUnavailable(_)))); + let session = session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); + } + #[tokio::test] async fn map_fetch_writes_cache_and_sets_ready_after_commit() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); diff --git a/backend/hls/src/media_reserve.rs b/backend/hls/src/media_reserve.rs index 4ef0849d9..cd482f9a5 100644 --- a/backend/hls/src/media_reserve.rs +++ b/backend/hls/src/media_reserve.rs @@ -3,7 +3,7 @@ use super::{ terminal_tail::{HlsEncryptionSignature, HlsMapSignature, HlsMediaContainer}, HlsAccessLeaseId, TransientManifestGeneration, HLS_ACCESS_LEASE_ID_PLACEHOLDER, }; -use std::{borrow::Cow, collections::BTreeSet, sync::Arc}; +use std::{borrow::Cow, collections::BTreeMap, sync::Arc}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsLeaseManifestSegment { @@ -121,7 +121,9 @@ pub struct HlsLeasePlaybackCursor { pub cursor_generation: u64, request_epoch: u64, highest_requested_proxy_seq: Option, - out_of_order_completed_proxy_seqs: BTreeSet, + first_requested_media_start_ms: Option, + highest_contiguous_completed_media_end_ms: Option, + out_of_order_completed_proxy_seqs: BTreeMap>, } const MAX_OUT_OF_ORDER_COMPLETIONS: usize = 64; @@ -132,6 +134,7 @@ const MAX_OUT_OF_ORDER_COMPLETIONS: usize = 64; pub struct HlsPlaybackRequestToken { proxy_seq: u64, request_epoch: u64, + media_end_ms: Option, } /// Result of committing a fully delivered segment to a playback cursor. @@ -146,9 +149,16 @@ pub enum HlsPlaybackCompletionOutcome { } impl HlsLeasePlaybackCursor { - /// Earliest sequence whose duration is needed to align the current manifest - /// window with this lease's measured playback position. + /// Earliest sequence still needed for playback evidence. Completed history is + /// carried by the cursor's media clock; undelivered gaps must remain visible. pub fn ready_timeline_start_proxy_seq(&self, manifest_first_proxy_seq: u64) -> u64 { + if self.first_requested_media_start_ms.is_some() { + return self + .highest_contiguous_completed_proxy_seq + .and_then(|seq| seq.checked_add(1)) + .or(self.first_requested_proxy_seq) + .map_or(manifest_first_proxy_seq, |seq| seq.min(manifest_first_proxy_seq)); + } self.first_requested_proxy_seq.map_or(manifest_first_proxy_seq, |first_requested_proxy_seq| { first_requested_proxy_seq.min(manifest_first_proxy_seq) }) @@ -164,6 +174,8 @@ impl HlsLeasePlaybackCursor { self.first_segment_completed_at_ms = None; self.highest_contiguous_completed_proxy_seq = None; self.highest_requested_proxy_seq = Some(proxy_seq); + self.first_requested_media_start_ms = None; + self.highest_contiguous_completed_media_end_ms = None; self.out_of_order_completed_proxy_seqs.clear(); } else { self.first_requested_proxy_seq.get_or_insert(proxy_seq); @@ -177,7 +189,22 @@ impl HlsLeasePlaybackCursor { ); self.cursor_generation = self.cursor_generation.saturating_add(1); - HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch } + HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch, media_end_ms: None } + } + + /// Binds playback evidence to the immutable session media clock before cache entries can be collected. + pub(crate) fn bind_request_timing( + &mut self, + mut token: HlsPlaybackRequestToken, + segment: &crate::SegmentEntry, + ) -> HlsPlaybackRequestToken { + debug_assert_eq!(token.proxy_seq, segment.proxy_seq); + debug_assert_eq!(token.request_epoch, self.request_epoch); + if self.first_requested_proxy_seq == Some(token.proxy_seq) { + self.first_requested_media_start_ms.get_or_insert(segment.media_start_ms); + } + token.media_end_ms = Some(segment.media_start_ms.saturating_add(segment.duration_ms)); + token } /// Commits a token only after the corresponding response body was delivered completely. @@ -196,7 +223,7 @@ impl HlsLeasePlaybackCursor { return HlsPlaybackCompletionOutcome::BeforeCurrentPosition; } if self.highest_contiguous_completed_proxy_seq.is_some_and(|highest| token.proxy_seq <= highest) - || self.out_of_order_completed_proxy_seqs.contains(&token.proxy_seq) + || self.out_of_order_completed_proxy_seqs.contains_key(&token.proxy_seq) { return HlsPlaybackCompletionOutcome::Duplicate; } @@ -206,7 +233,7 @@ impl HlsLeasePlaybackCursor { .and_then(|highest| highest.checked_add(1)) .unwrap_or(first_requested_proxy_seq); if token.proxy_seq != next_expected_proxy_seq { - self.buffer_out_of_order_completion(token.proxy_seq); + self.buffer_out_of_order_completion(token); self.cursor_generation = self.cursor_generation.saturating_add(1); return HlsPlaybackCompletionOutcome::Buffered; } @@ -215,29 +242,29 @@ impl HlsLeasePlaybackCursor { self.first_segment_completed_at_ms = Some(completed_at_ms); } self.highest_contiguous_completed_proxy_seq = Some(token.proxy_seq); + self.highest_contiguous_completed_media_end_ms = token.media_end_ms; self.advance_over_buffered_completions(); self.cursor_generation = self.cursor_generation.saturating_add(1); HlsPlaybackCompletionOutcome::Advanced } - fn buffer_out_of_order_completion(&mut self, proxy_seq: u64) { - self.out_of_order_completed_proxy_seqs.insert(proxy_seq); + fn buffer_out_of_order_completion(&mut self, token: HlsPlaybackRequestToken) { + self.out_of_order_completed_proxy_seqs.insert(token.proxy_seq, token.media_end_ms); if self.out_of_order_completed_proxy_seqs.len() <= MAX_OUT_OF_ORDER_COMPLETIONS { return; } - if let Some(furthest_proxy_seq) = self.out_of_order_completed_proxy_seqs.last().copied() { - self.out_of_order_completed_proxy_seqs.remove(&furthest_proxy_seq); - } + self.out_of_order_completed_proxy_seqs.pop_last(); } fn advance_over_buffered_completions(&mut self) { while let Some(next_proxy_seq) = self.highest_contiguous_completed_proxy_seq.and_then(|highest| highest.checked_add(1)) { - if !self.out_of_order_completed_proxy_seqs.remove(&next_proxy_seq) { + let Some(media_end_ms) = self.out_of_order_completed_proxy_seqs.remove(&next_proxy_seq) else { break; - } + }; self.highest_contiguous_completed_proxy_seq = Some(next_proxy_seq); + self.highest_contiguous_completed_media_end_ms = media_end_ms; } } } @@ -311,7 +338,12 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv cutover_required: input.origin_path_degraded && !input.recovery_committed, }; } - let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest); + let first_needed_seq = if input.cursor.first_requested_media_start_ms.is_some() { + input.cursor.ready_timeline_start_proxy_seq(input.manifest.first_proxy_seq) + } else { + input.manifest.first_proxy_seq + }; + let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest, first_needed_seq); let visible_tail = aligned_visible_tail_ms(input.ready_timeline, input.manifest); let guaranteed_media_horizon_ms = guaranteed_ready_span.horizon_ms; let conservative_playback_position_ms = @@ -353,9 +385,11 @@ struct HlsGuaranteedReadySpan { fn contiguous_ready_span( timeline: &HlsReadyTimelineSnapshot, manifest: &HlsLeaseManifestSnapshot, + first_proxy_seq: u64, ) -> HlsGuaranteedReadySpan { - let fallback_start_ms = manifest_visible_start_ms(manifest); - let Some(start_index) = timeline.units.iter().position(|unit| unit.proxy_seq == manifest.first_proxy_seq) else { + let fallback_start_ms = + if first_proxy_seq == manifest.first_proxy_seq { manifest_visible_start_ms(manifest) } else { 0 }; + let Some(start_index) = timeline.units.iter().position(|unit| unit.proxy_seq == first_proxy_seq) else { return HlsGuaranteedReadySpan { start_ms: fallback_start_ms, horizon_ms: fallback_start_ms, @@ -376,7 +410,7 @@ fn contiguous_ready_span( }; } - let mut expected_seq = manifest.first_proxy_seq; + let mut expected_seq = first_proxy_seq; let mut expected_start_ms = first_unit.start_ms; let mut horizon_ms = first_unit.start_ms; let mut end_unit_index = start_index; @@ -436,21 +470,29 @@ fn conservative_playback_position( let Some(first_seq) = input.cursor.first_requested_proxy_seq else { return visible_tail_ms; }; - let Some(first_unit) = input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq) else { + let Some(first_start_ms) = input.cursor.first_requested_media_start_ms.or_else(|| { + input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq).map(|unit| unit.start_ms) + }) else { return visible_tail_ms.max(guaranteed_media_horizon_ms); }; if input.cursor.first_segment_completed_at_ms.is_none() { - return visible_tail_ms.max(first_unit.start_ms); + return visible_tail_ms.max(first_start_ms); } let delivered_end = input .cursor - .highest_contiguous_completed_proxy_seq - .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) - .map_or(guaranteed_media_horizon_ms, |unit| unit.start_ms.saturating_add(unit.duration_ms)); + .highest_contiguous_completed_media_end_ms + .or_else(|| { + input + .cursor + .highest_contiguous_completed_proxy_seq + .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) + .map(|unit| unit.start_ms.saturating_add(unit.duration_ms)) + }) + .unwrap_or(guaranteed_media_horizon_ms); let elapsed_guarded = input.cursor.first_segment_completed_at_ms.map_or(0, |completed_at_ms| { input.now_ms.saturating_sub(completed_at_ms).saturating_mul(u64::from(input.playback_rate_guard_milli)) / 1_000 }); - delivered_end.min(first_unit.start_ms.saturating_add(elapsed_guarded)) + delivered_end.min(first_start_ms.saturating_add(elapsed_guarded)) } fn guaranteed_transition_margin( @@ -539,7 +581,8 @@ pub fn evaluate_startup_admission(input: HlsStartupAdmissionInput<'_>) -> HlsSta decision, }; } - let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest); + let guaranteed_ready_span = + contiguous_ready_span(input.ready_timeline, input.manifest, input.manifest.first_proxy_seq); let visible_tail_ms = aligned_visible_tail_ms(input.ready_timeline, input.manifest); let visible_contiguous_ready_duration_ms = guaranteed_ready_span.horizon_ms.min(visible_tail_ms).saturating_sub(guaranteed_ready_span.start_ms); @@ -693,6 +736,31 @@ mod tests { }) } + #[test] + fn one_and_two_segment_playlists_admit_only_actual_ready_duration_and_claim_no_hidden_reserve() { + for count in [1, 2] { + let mut short = manifest(); + short.visible_segments = Arc::from(&short.visible_segments[..count]); + short.last_proxy_seq = u64::try_from(count - 1).unwrap(); + short.playlist_duration_ms = short.visible_segments.iter().map(|segment| segment.duration_ms).sum(); + short.last_visible_media_end_ms = short.playlist_duration_ms; + let ready = ready_timeline(); + let mut timeline = HlsReadyTimelineSnapshot { units: Arc::from(&ready.units[..count]) }; + let admitted = admission(&short, &timeline, HlsStartupAdmissionOriginState::Healthy, 0); + assert_eq!(admitted.decision, HlsStartupAdmissionDecision::Admit); + assert_eq!(admitted.visible_contiguous_ready_duration_ms, short.playlist_duration_ms); + assert_eq!(admitted.prospective_lease_reserve_ms, 0); + let reserve = reserve(&short, &HlsLeasePlaybackCursor::default(), &timeline, 0); + assert_eq!(reserve.initial_hidden_ready_duration_ms, 0); + assert_eq!(reserve.guaranteed_media_horizon_ms, short.playlist_duration_ms); + Arc::make_mut(&mut timeline.units)[count - 1].state = HlsReadyMediaState::NotReady; + assert_eq!( + admission(&short, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(HlsStartupAdmissionRejection::InsufficientVisibleReadyMedia) + ); + } + } + #[test] fn strip_reserve_is_sum_of_actual_hidden_ready_durations() { let reserve = reserve(&manifest(), &HlsLeasePlaybackCursor::default(), &ready_timeline(), 0); @@ -1211,4 +1279,98 @@ mod tests { HlsStartupAdmissionDecision::Reject(HlsStartupAdmissionRejection::InsufficientVisibleReadyMedia) ); } + + fn variable_duration_session() -> crate::HlsSession { + use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + let OriginManifestParseOutcome::Normal(origin) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:9\n#EXT-X-MEDIA-SEQUENCE:0\n\ + #EXTINF:4,\n0.ts\n#EXTINF:5,\n1.ts\n#EXTINF:6,\n2.ts\n#EXTINF:3,\n3.ts\n#EXTINF:9,\n4.ts\n", + "http://origin.example/index.m3u8", + ) else { + panic!("variable timeline parses"); + }; + let mut session = crate::HlsSession::new(crate::HlsSessionKey::new(1, "variable-clock"), b"secret", 0); + session.apply_origin_manifest(&origin).expect("variable timeline maps"); + for segment in session.segments.values_mut() { + segment.status = crate::SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 }; + } + session + } + + #[test] + fn timed_out_of_order_completions_preserve_variable_duration_anchor_after_collection() { + let mut session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let tokens = (0..3) + .map(|seq| { + let token = cursor.record_request_started(seq, 1_000); + cursor.bind_request_timing(token, &session.segments[&seq]) + }) + .collect::>(); + assert_eq!(cursor.record_request_completed(tokens[2], 1_100), HlsPlaybackCompletionOutcome::Buffered); + assert_eq!(cursor.record_request_completed(tokens[0], 1_200), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(4_000)); + assert_eq!(cursor.record_request_completed(tokens[1], 1_300), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(15_000)); + let mut current = manifest(); + current.first_proxy_seq = 2; + current.last_proxy_seq = 2; + current.visible_segments = Arc::from([current.visible_segments[2].clone()]); + current.playlist_duration_ms = 6_000; + current.last_visible_media_end_ms = 6_000; + let before = reserve(¤t, &cursor, &session.ready_timeline_snapshot(2, 11_200), 11_200); + session.segments.remove(&0); + session.segments.remove(&1); + let after = reserve( + ¤t, + &cursor, + &session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(current.first_proxy_seq), 11_200), + 11_200, + ); + assert_eq!(before, after); + assert_eq!(after.conservative_playback_position_ms, 10_500); + assert_eq!(after.guaranteed_reserve_ms, 16_500); + } + + #[test] + fn timed_cursor_does_not_bridge_an_undelivered_gap_before_the_new_manifest_window() { + let mut session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let first = cursor.record_request_started(0, 1_000); + let first = cursor.bind_request_timing(first, &session.segments[&0]); + assert_eq!(cursor.record_request_completed(first, 1_000), HlsPlaybackCompletionOutcome::Advanced); + let mut current = manifest(); + current.first_proxy_seq = 2; + current.last_proxy_seq = 2; + current.visible_segments = Arc::from([current.visible_segments[2].clone()]); + current.playlist_duration_ms = 6_000; + current.last_visible_media_end_ms = 6_000; + assert_eq!(cursor.ready_timeline_start_proxy_seq(2), 1); + session.segments.remove(&0); + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 25_950); + session.segments.get_mut(&1).expect("undelivered segment").status = crate::SegmentCacheStatus::Discovered; + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 2_950); + session.segments.remove(&1); + let timeline = session.ready_timeline_snapshot(1, 2_000); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 0); + } + + #[test] + fn timed_forward_seek_discards_the_previous_media_anchor_and_late_completion() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let old = cursor.record_request_started(0, 1_000); + let old = cursor.bind_request_timing(old, &session.segments[&0]); + let seek = cursor.record_request_started(4, 2_000); + let seek = cursor.bind_request_timing(seek, &session.segments[&4]); + assert_eq!(cursor.record_request_completed(old, 2_500), HlsPlaybackCompletionOutcome::StaleRequest); + assert_eq!(cursor.record_request_completed(seek, 3_000), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.first_requested_media_start_ms, Some(18_000)); + assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(27_000)); + let reserve = reserve(&manifest(), &cursor, &session.ready_timeline_snapshot(0, 4_000), 4_000); + assert_eq!(reserve.conservative_playback_position_ms, 19_050); + assert_eq!(reserve.guaranteed_reserve_ms, 7_950); + } } diff --git a/backend/hls/src/observability.rs b/backend/hls/src/observability.rs index 7d9ae134f..f90367d0f 100644 --- a/backend/hls/src/observability.rs +++ b/backend/hls/src/observability.rs @@ -21,6 +21,10 @@ pub use tuliprox_core::utils::content_coding::{ #[derive(Debug, Default)] pub struct HlsCacheMetrics { revision_disk_bytes: AtomicU64, + revision_deletions_planned: AtomicU64, + revision_deletions_succeeded: AtomicU64, + revision_deletions_failed: AtomicU64, + revision_bytes_reclaimed: AtomicU64, progressive_replay_bytes: AtomicU64, progressive_active_fills: AtomicU64, deferred_repairs: AtomicU64, @@ -61,6 +65,10 @@ pub struct HlsCacheMetrics { #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct HlsCacheMetricsSnapshot { pub revision_disk_bytes: u64, + pub revision_deletions_planned: u64, + pub revision_deletions_succeeded: u64, + pub revision_deletions_failed: u64, + pub revision_bytes_reclaimed: u64, pub progressive_replay_bytes: u64, pub progressive_active_fills: u64, pub deferred_repairs: u64, @@ -104,6 +112,13 @@ impl HlsCacheMetrics { self.progressive_replay_bytes.store(u64::try_from(bytes).unwrap_or(u64::MAX), Ordering::Relaxed); self.deferred_repairs.store(u64::try_from(repairs).unwrap_or(u64::MAX), Ordering::Relaxed); } + pub(crate) fn record_revision_gc(&self, planned: usize, succeeded: usize, failed: usize, bytes: u64) { + self.revision_deletions_planned.fetch_add(u64::try_from(planned).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_deletions_succeeded.fetch_add(u64::try_from(succeeded).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_deletions_failed.fetch_add(u64::try_from(failed).unwrap_or(u64::MAX), Ordering::Relaxed); + self.revision_bytes_reclaimed.fetch_add(bytes, Ordering::Relaxed); + } + pub fn record_revision_disk_bytes(&self, bytes: u64) { self.revision_disk_bytes.store(bytes, Ordering::Relaxed); } pub fn record_session_created(&self) { increment(&self.sessions_created, 1); } pub fn record_session_reused(&self) { increment(&self.sessions_reused, 1); } @@ -157,6 +172,10 @@ impl HlsCacheMetrics { pub fn snapshot(&self) -> HlsCacheMetricsSnapshot { HlsCacheMetricsSnapshot { revision_disk_bytes: load(&self.revision_disk_bytes), + revision_deletions_planned: load(&self.revision_deletions_planned), + revision_deletions_succeeded: load(&self.revision_deletions_succeeded), + revision_deletions_failed: load(&self.revision_deletions_failed), + revision_bytes_reclaimed: load(&self.revision_bytes_reclaimed), progressive_replay_bytes: load(&self.progressive_replay_bytes), progressive_active_fills: load(&self.progressive_active_fills), deferred_repairs: load(&self.deferred_repairs), diff --git a/backend/hls/src/origin.rs b/backend/hls/src/origin.rs index e18872fc3..7144201ee 100644 --- a/backend/hls/src/origin.rs +++ b/backend/hls/src/origin.rs @@ -227,7 +227,15 @@ impl fmt::Debug for HlsOriginAccountIoLeaseGuard { } impl HlsOriginAccountIoLeaseGuard { - pub fn new(binding: HlsOriginAccountBinding, lease: Arc) -> Self { + pub fn new(mut binding: HlsOriginAccountBinding, lease: Arc) -> Self { + { + let handle = lease.provider_handle.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(handle) = handle.as_ref() { + // A joining caller may still carry the expired predecessor's lease metadata. + binding.playback_request_id = handle.playback_request_id; + binding.provider_binding_tag = handle.binding_tag; + } + } Self { binding, lease, disarmed: false } } @@ -512,12 +520,14 @@ pub enum HlsOriginAccountStatus { Known, Missing, Expired, + Blocked, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HlsBoundAccountAcquireErrorKind { Missing, Expired, + Blocked, Exhausted, ReservedForOther, Detached, @@ -528,7 +538,7 @@ pub enum HlsBoundAccountAcquireErrorKind { } impl HlsBoundAccountAcquireErrorKind { - pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired) } + pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired | Self::Blocked) } pub fn is_retryable_resource_failure(self) -> bool { // A competing session or an in-flight binding handoff can temporarily @@ -549,6 +559,7 @@ impl HlsBoundAccountAcquireErrorKind { match self { Self::Missing => "Missing", Self::Expired => "Expired", + Self::Blocked => "Blocked", Self::Exhausted => "Exhausted", Self::ReservedForOther => "ReservedForOther", Self::Detached => "Detached", @@ -634,6 +645,9 @@ pub struct HlsOriginIoContext { pub reservation_ttl_secs: u64, pub preacquired_provider_handle: Option>>>, pub started_generation: Option, + pub prepared_account: Option, + pub playback_kind: PlaybackKind, + pub account_runtime: Option>, } impl HlsOriginIoContext { @@ -649,6 +663,11 @@ impl HlsOriginIoContext { self } + pub fn with_managed_provider_handle(mut self, handle: tuliprox_session::ManagedProviderHandle) -> Self { + self.preacquired_provider_handle = Some(Arc::new(std::sync::Mutex::new(Some(handle)))); + self + } + pub fn take_preacquired_provider_handle(&self) -> Option { let handle = self.preacquired_provider_handle.as_ref()?; let mut guard = match handle.lock() { @@ -673,6 +692,9 @@ pub fn hls_origin_account_status(ctx: &HlsCtx, binding: &HlsOriginAccountBinding if is_input_expired(provider_config.exp_date()) { return HlsOriginAccountStatus::Expired; } + if provider_config.is_account_blocked() { + return HlsOriginAccountStatus::Blocked; + } HlsOriginAccountStatus::Known } @@ -683,6 +705,7 @@ pub fn acquire_bound_hls_origin_account_handle( allow_grace: bool, priority: i8, connection_kind: ConnectionKind, + playback_kind: PlaybackKind, ) -> Result { if binding.is_detached() { return Err(HlsBoundAccountAcquireErrorKind::Detached); @@ -690,6 +713,7 @@ pub fn acquire_bound_hls_origin_account_handle( match hls_origin_account_status(ctx, binding) { HlsOriginAccountStatus::Missing => return Err(HlsBoundAccountAcquireErrorKind::Missing), HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), + HlsOriginAccountStatus::Blocked => return Err(HlsBoundAccountAcquireErrorKind::Blocked), HlsOriginAccountStatus::Known => {} } if ctx.active_provider.is_provider_reserved_for_other_session(&binding.account_name, Some(&binding.session_owner)) { @@ -701,7 +725,7 @@ pub fn acquire_bound_hls_origin_account_handle( allow_grace, priority, connection_kind, - Some(PlaybackLeaseRef::new(&binding.session_owner, PlaybackKind::LiveHls)), + Some(PlaybackLeaseRef::new(&binding.session_owner, playback_kind)), ); if let Some(handle) = handle { return Ok(handle); @@ -750,6 +774,13 @@ async fn begin_hls_origin_account_io_inner( let slot = reserve_hls_origin_account_io_slot(session, binding, deadline).await?; if let HlsOriginAccountIoSlot::Joined(lease) = slot { let guard = HlsOriginAccountIoLeaseGuard::new(binding.clone(), lease); + match hls_origin_account_status(&origin_io.ctx, binding) { + HlsOriginAccountStatus::Known => {} + HlsOriginAccountStatus::Missing => return Err(HlsBoundAccountAcquireErrorKind::Missing), + HlsOriginAccountStatus::Expired => return Err(HlsBoundAccountAcquireErrorKind::Expired), + HlsOriginAccountStatus::Blocked => return Err(HlsBoundAccountAcquireErrorKind::Blocked), + } + // Release any unused pre-acquired handle synchronously on drop. It needs no // cleanup permit, so a join cannot self-block on a single-permit cleanup queue. drop(origin_io.take_preacquired_provider_handle()); @@ -766,6 +797,12 @@ async fn begin_hls_origin_account_io_inner( let cleanup_permit = reserve_cleanup_permit(origin_io, deadline).await?; let acquired_handle = if let Some(mut managed) = origin_io.take_preacquired_provider_handle() { + if !managed.handle().is_some_and(|handle| { + handle.allocation.get_provider_name().as_ref() == Some(&binding.account_name) + && binding.provider_binding_tag.is_none_or(|tag| handle.binding_tag == Some(tag)) + }) { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } // Disarm the managed owner only after the cleanup permit is already reserved, // so the naked handle is immediately paired with its release responsibility. // The wrapper always carries its handle; fall back to a fresh acquire defensively. @@ -778,6 +815,7 @@ async fn begin_hls_origin_account_io_inner( origin_io.allow_grace, origin_io.priority, origin_io.connection_kind, + origin_io.playback_kind, ), } } else { @@ -788,6 +826,7 @@ async fn begin_hls_origin_account_io_inner( origin_io.allow_grace, origin_io.priority, origin_io.connection_kind, + origin_io.playback_kind, ) }; @@ -939,7 +978,7 @@ async fn store_acquired_hls_origin_account_io_handle( pending_guard.disarm(); return Err(HlsBoundAccountAcquireErrorKind::AcquireTimedOut); } - let lease_ref = PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }; + let lease_ref = PlaybackLeaseRef { owner: &binding.session_owner, kind: origin_io.playback_kind, request_id }; origin_io.ctx.active_provider.refresh_playback_lease( &binding.account_name, &lease_ref, @@ -1067,7 +1106,11 @@ pub async fn finish_hls_origin_account_io( if let Some(request_id) = binding.playback_request_id { origin_io.ctx.active_provider.refresh_playback_lease( &binding.account_name, - &PlaybackLeaseRef { owner: &binding.session_owner, kind: PlaybackKind::LiveHls, request_id }, + &PlaybackLeaseRef { + owner: &binding.session_owner, + kind: session.origin_playback_kind(), + request_id, + }, confirmed_ttl_secs, ); } @@ -1347,6 +1390,7 @@ mod tests { fn bound_account_acquire_error_rebind_policy_matches_concept() { assert!(HlsBoundAccountAcquireErrorKind::Missing.allows_rebind()); assert!(HlsBoundAccountAcquireErrorKind::Expired.allows_rebind()); + assert!(HlsBoundAccountAcquireErrorKind::Blocked.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::Exhausted.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::ReservedForOther.allows_rebind()); assert!(!HlsBoundAccountAcquireErrorKind::Detached.allows_rebind()); @@ -1581,6 +1625,9 @@ mod tests { reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: None, + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let mut session = HlsSession::new(HlsSessionKey::new(1, "permit-timeout"), b"secret", 1_000); diff --git a/backend/hls/src/origin_account_runtime.rs b/backend/hls/src/origin_account_runtime.rs new file mode 100644 index 000000000..23b419cc9 --- /dev/null +++ b/backend/hls/src/origin_account_runtime.rs @@ -0,0 +1,205 @@ +use crate::{ + manifest_fetch::LiveHlsOriginEntry, HlsBoundAccountAcquireErrorKind, HlsOriginIoContext, HlsOriginWorkClass, + HlsSessionHandle, +}; +use std::{fmt, future::Future, pin::Pin}; + +/// Origin operation being prepared; media work must first qualify an account handoff. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HlsOriginAccountWork { + Manifest, + Media, +} + +/// Account preparation result resolved against the current source configuration. +pub struct HlsPreparedOriginAccount { + pub entry: LiveHlsOriginEntry, + pub snapshot: HlsOriginAccountSnapshot, +} + +/// Account identity, work generation and provider headers captured together with a resolved entry URL. +#[derive(Clone)] +pub struct HlsOriginAccountSnapshot { + pub binding: Option, + pub generation: u64, + pub provider_session_headers: axum::http::HeaderMap, +} + +impl HlsOriginAccountSnapshot { + pub fn from_session(session: &crate::HlsSession) -> Self { + Self { + binding: session.origin_account_binding.clone(), + generation: session.activity.origin_work_generation, + provider_session_headers: session.origin_provider_session_headers.clone(), + } + } + + pub fn matches_session(&self, session: &crate::HlsSession) -> bool { + self.generation == session.activity.origin_work_generation + && match (&self.binding, &session.origin_account_binding) { + (Some(expected), Some(current)) => { + expected.input_name == current.input_name + && expected.account_name == current.account_name + && expected.session_owner == current.session_owner + && expected.generation == current.generation + && expected.provider_binding_tag == current.provider_binding_tag + && expected.is_active() == current.is_active() + } + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } + } +} + +/// App-owned account allocation/URL adapter used by existing HLS workers. +pub trait HlsOriginAccountRuntime: Send + Sync + fmt::Debug { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + work: HlsOriginAccountWork, + class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>>; +} + +pub(crate) async fn prepare_media_origin( + origin_io: Option<&HlsOriginIoContext>, + session: &HlsSessionHandle, +) -> Result<(), HlsBoundAccountAcquireErrorKind> { + let Some(io) = origin_io else { + return Ok(()); + }; + let Some(runtime) = io.account_runtime.as_ref() else { + return Ok(()); + }; + let class = if io.allow_grace { HlsOriginWorkClass::Demand } else { HlsOriginWorkClass::Background }; + let deadline = tokio::time::Instant::now() + io.ctx.hls_proxy.segment_fetch_policy().origin_object_wait_timeout(); + tokio::time::timeout_at(deadline, async { + loop { + match runtime.prepare(session, HlsOriginAccountWork::Media, class).await { + Ok(_) => return Ok(()), + Err(error) => { + if class != HlsOriginWorkClass::Background { + return Err(error); + } + let now_ms = tuliprox_core::utils::current_time_millis(); + let retry_at = session.read().await.origin_account_rebind.next_rebind_allowed_at_ms; + let Some(wait_ms) = + retry_at.and_then(|retry_at| retry_at.checked_sub(now_ms)).filter(|wait| *wait > 0) + else { + return Err(error); + }; + tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await; + } + } + } + }) + .await + .map_err(|_| HlsBoundAccountAcquireErrorKind::WaitTimedOut)? +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{HlsCtx, HlsSession, HlsSessionKey}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + + #[derive(Clone, Copy, Debug)] + enum PrepareBehavior { + RetryOnce, + Stall, + } + + #[derive(Debug)] + struct TestRuntime { + attempts: AtomicUsize, + behavior: PrepareBehavior, + } + + impl HlsOriginAccountRuntime for TestRuntime { + fn prepare<'a>( + &'a self, + session: &'a HlsSessionHandle, + _work: HlsOriginAccountWork, + _class: HlsOriginWorkClass, + ) -> Pin> + Send + 'a>> + { + Box::pin(async move { + let attempt = self.attempts.fetch_add(1, Ordering::AcqRel); + match self.behavior { + PrepareBehavior::Stall => std::future::pending().await, + PrepareBehavior::RetryOnce if attempt == 0 => { + session.write().await.origin_account_rebind.mark_attempt_started( + Arc::from("unavailable"), + tuliprox_core::utils::current_time_millis(), + ); + Err(HlsBoundAccountAcquireErrorKind::Unavailable) + } + PrepareBehavior::RetryOnce => Ok(HlsPreparedOriginAccount { + entry: LiveHlsOriginEntry::parse("https://origin.example/index.m3u8").unwrap(), + snapshot: HlsOriginAccountSnapshot::from_session(&*session.read().await), + }), + } + }) + } + } + + fn fixture(behavior: PrepareBehavior) -> (HlsOriginIoContext, HlsSessionHandle, Arc) { + let runtime = Arc::new(TestRuntime { attempts: AtomicUsize::new(0), behavior }); + let io = HlsOriginIoContext { + ctx: HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }; + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "runtime-backoff"), b"secret", 0))); + (io, session, runtime) + } + + #[tokio::test(start_paused = true)] + async fn background_prepare_waits_for_rebind_backoff_before_retrying() { + let (io, session, runtime) = fixture(PrepareBehavior::RetryOnce); + let mut prepare = Box::pin(prepare_media_origin(Some(&io), &session)); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + tokio::time::advance(std::time::Duration::from_secs(1)).await; + assert!(futures::poll!(prepare.as_mut()).is_pending()); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + tokio::time::advance(std::time::Duration::from_secs(2)).await; + prepare.await.unwrap(); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 2); + } + + #[tokio::test(start_paused = true)] + async fn stalled_background_prepare_is_bounded_by_the_existing_object_budget() { + let (io, session, runtime) = fixture(PrepareBehavior::Stall); + let budget = io.ctx.hls_proxy.segment_fetch_policy().origin_object_wait_timeout(); + let mut prepare = Box::pin(prepare_media_origin(Some(&io), &session)); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + tokio::time::advance(budget).await; + assert_eq!(prepare.await, Err(HlsBoundAccountAcquireErrorKind::WaitTimedOut)); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + } + + #[tokio::test(start_paused = true)] + async fn interactive_prepare_returns_temporary_failure_without_background_waiting() { + let (io, session, runtime) = fixture(PrepareBehavior::RetryOnce); + let io = io.with_grace(true); + assert_eq!(prepare_media_origin(Some(&io), &session).await, Err(HlsBoundAccountAcquireErrorKind::Unavailable)); + assert_eq!(runtime.attempts.load(Ordering::Acquire), 1); + assert!(session.read().await.origin_account_rebind.next_rebind_allowed_at_ms.is_some()); + } +} diff --git a/backend/hls/src/origin_attempt.rs b/backend/hls/src/origin_attempt.rs new file mode 100644 index 000000000..64cca1657 --- /dev/null +++ b/backend/hls/src/origin_attempt.rs @@ -0,0 +1,360 @@ +use crate::{ + begin_hls_origin_account_io_bounded, finish_hls_origin_account_io, origin_account_runtime::prepare_media_origin, + HlsBoundAccountAcquireErrorKind, HlsOriginAccountBinding, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext, + HlsSession, HlsSessionHandle, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tuliprox_core::utils::current_time_millis; + +/// Whether the completed media attempt still belongs to the current account. +pub(crate) struct HlsMediaOriginWorkFinish { + pub(crate) generation_valid: bool, +} + +/// Owns the work counter and provider lease for one generation-bound media attempt. +pub(crate) struct HlsMediaOriginAttemptGuard { + generation: u64, + binding: Option, + active_origin_work_count: Option>, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +} + +impl HlsMediaOriginAttemptGuard { + fn matches_session(&self, session: &HlsSession) -> bool { + self.generation == session.activity.origin_work_generation + && match (&self.binding, &session.origin_account_binding) { + (Some(expected), Some(current)) => { + let expected_tag = self + .provider_lease + .as_ref() + .map_or(expected.provider_binding_tag, |(_, lease)| lease.binding().provider_binding_tag); + expected.input_name == current.input_name + && expected.account_name == current.account_name + && expected.session_owner == current.session_owner + && expected.generation == current.generation + && expected.is_active() == current.is_active() + // Reacquiring an expired provider lease may renew its tag without an account handoff. + && expected_tag.is_none_or(|tag| current.provider_binding_tag == Some(tag)) + } + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } + } + + pub(crate) async fn finish(mut self, session: &HlsSessionHandle) -> HlsMediaOriginWorkFinish { + let (generation_valid, refresh_reservation) = { + let mut session = session.write().await; + let generation_valid = self.matches_session(&session); + if self.active_origin_work_count.take().is_some() { + session.finish_origin_work(self.generation); + } + (generation_valid, generation_valid && session.should_refresh_origin_reservation(current_time_millis())) + }; + if let Some((origin_io, lease)) = self.provider_lease.take() { + finish_hls_origin_account_io(&origin_io, session, lease, refresh_reservation).await; + } + HlsMediaOriginWorkFinish { generation_valid } + } +} + +impl Drop for HlsMediaOriginAttemptGuard { + fn drop(&mut self) { + if let Some(count) = self.active_origin_work_count.take() { + count.fetch_sub(1, Ordering::AcqRel); + } + } +} + +/// Captures work and binding atomically, then rejects a handoff during provider acquisition. +pub(crate) async fn prepare_media_origin_attempt( + origin_io: Option<&HlsOriginIoContext>, + session: &HlsSessionHandle, + expected_generation: u64, + acquire_timeout: Duration, +) -> Result { + prepare_media_origin(origin_io, session).await?; + let mut attempt = { + let mut session = session.write().await; + if session.activity.origin_work_generation != expected_generation { + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + let binding = session.origin_account_binding.clone(); + if origin_io.is_some() && binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached) { + return Err(HlsBoundAccountAcquireErrorKind::Detached); + } + let active_origin_work_count = origin_io.map(|_| { + session.start_origin_work(); + Arc::clone(&session.activity.active_origin_work_count) + }); + HlsMediaOriginAttemptGuard { + generation: expected_generation, + binding, + active_origin_work_count, + provider_lease: None, + } + }; + let (Some(origin_io), Some(binding)) = (origin_io, attempt.binding.as_ref()) else { + return Ok(attempt); + }; + let acquired = begin_hls_origin_account_io_bounded(origin_io, session, binding, acquire_timeout).await; + let error = match acquired { + Ok(lease) => { + attempt.provider_lease = Some((origin_io.clone(), lease)); + None + } + Err(error) => Some(error), + }; + let still_current = attempt.matches_session(&*session.read().await); + if !still_current { + attempt.finish(session).await; + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + if let Some(error) = error { + attempt.finish(session).await; + if error.allows_rebind() { + let _ = prepare_media_origin(Some(origin_io), session).await; + return Err(HlsBoundAccountAcquireErrorKind::StoreRace); + } + return Err(error); + } + Ok(attempt) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::prepare_media_origin_attempt; + use crate::{ + HlsBoundAccountAcquireErrorKind, HlsCtx, HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginIoContext, + HlsSession, HlsSessionHandle, HlsSessionKey, + }; + use shared::model::{ConfigInputAliasDto, ConfigInputDto}; + use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, + }; + use tuliprox_core::model::{Config, ConfigInput, PlaybackKind, SourcesConfig}; + use tuliprox_session::ConnectionKind; + + pub(crate) async fn bound_origin_io(session: &HlsSessionHandle) -> (HlsOriginIoContext, HlsOriginAccountBinding) { + let ctx = HlsCtx::for_test(Config::default()); + let input = ConfigInput::from(&ConfigInputDto { + name: Arc::from("account-a"), + url: "http://origin.example/account-a/index.m3u8".to_string(), + enabled: true, + max_connections: 2, + aliases: Some(vec![ConfigInputAliasDto { + name: Arc::from("account-b"), + url: "http://origin.example/account-b/index.m3u8".to_string(), + enabled: true, + max_connections: 2, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }); + ctx.app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(input)], ..SourcesConfig::default() })); + ctx.active_provider.update_config(&ctx.app_config); + let replacement = { + let mut session = session.write().await; + let binding = HlsOriginAccountBinding::new( + Arc::from("account-a"), + Arc::from("account-a"), + &session.proxy_session_id, + 1_000, + ); + let replacement = HlsOriginAccountBinding::rebound( + Arc::clone(&binding.input_name), + Arc::from("account-b"), + binding.session_owner.clone(), + binding.generation + 1, + 2_000, + ); + session.replace_origin_account_binding(Some(binding)); + replacement + }; + let io = HlsOriginIoContext { + ctx, + client_addr: "127.0.0.1:12345".parse().expect("test address"), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: None, + }; + (io, replacement) + } + + #[tokio::test] + async fn handoff_after_lease_acquisition_releases_the_old_attempt_without_touching_its_successor() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "lease-handoff"), b"secret", 0))); + let (io, replacement) = bound_origin_io(&session).await; + let original_lease = { + let mut session = session.write().await; + let lease = Arc::new(HlsOriginAccountIoLease::active_for_test( + session.origin_account_binding.as_ref().expect("original binding"), + 1, + )); + session.origin_account_io_lease = Some(Arc::clone(&lease)); + lease + }; + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let generation = session.read().await.activity.origin_work_generation; + let read_blocker = session.read().await; + let mut prepare = + Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + // Queue a reader between the atomic start and the provider-slot write. + let mut between_start_and_lease = Box::pin(session.read()); + assert!(futures::poll!(between_start_and_lease.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + let between_start_and_lease = between_start_and_lease.await; + let mut handoff = Box::pin(session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(between_start_and_lease); + assert!(futures::poll!(prepare.as_mut()).is_pending()); + { + let mut session = handoff.await; + assert_eq!( + original_lease.active_io_count.load(Ordering::Acquire), + 2, + "old lease was acquired before handoff" + ); + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + assert!(matches!(prepare.await, Err(HlsBoundAccountAcquireErrorKind::StoreRace))); + let session = session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(original_lease.active_io_count.load(Ordering::Acquire), 1); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); + assert!(session.origin_account_io_lease.as_ref().is_some_and(|lease| Arc::ptr_eq(lease, &replacement_lease))); + assert!(session.origin_account_binding.as_ref().expect("new binding").last_origin_io_at_ms.is_none()); + } + + #[tokio::test] + async fn initial_provider_tag_assignment_keeps_the_media_attempt_current() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "initial-tag"), b"secret", 0))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let attempt = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("initial binding can acquire its first provider lease"); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 1); + assert!(attempt.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn same_account_reacquisition_after_provider_lease_expiry_keeps_the_media_attempt_current() { + let session = + Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "lease-expiry"), b"secret", 0))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let first = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("first lease acquisition"); + assert!(first.finish(&session).await.generation_valid); + let original = session.read().await.origin_account_binding.clone().expect("original binding"); + let original_tag = original.provider_binding_tag.expect("first lease has a provider tag"); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + + tokio::time::advance(Duration::from_secs(tuliprox_session::provider_leases::STARTING_LEASE_TTL_SECS + 1)).await; + + let reacquired = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("same account reacquires its expired provider lease"); + { + let session = session.read().await; + let current = session.origin_account_binding.as_ref().expect("reacquired binding"); + assert_eq!( + (¤t.input_name, ¤t.account_name, ¤t.session_owner, current.generation), + (&original.input_name, &original.account_name, &original.session_owner, original.generation) + ); + assert_eq!(session.activity.origin_work_generation, generation); + assert_ne!(current.provider_binding_tag.expect("new lease has a provider tag"), original_tag); + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 1); + } + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(reacquired.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn concurrent_media_attempts_join_the_reacquired_provider_lease_with_its_current_tag() { + let session = Arc::new(tokio::sync::RwLock::new(HlsSession::new( + HlsSessionKey::new(1, "concurrent-lease-expiry"), + b"secret", + 0, + ))); + let (io, _) = bound_origin_io(&session).await; + let generation = session.read().await.activity.origin_work_generation; + let baseline = prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1)) + .await + .expect("baseline provider lease"); + assert!(baseline.finish(&session).await.generation_valid); + let original_tag = session.read().await.origin_account_binding.as_ref().expect("binding").provider_binding_tag; + tokio::time::advance(Duration::from_secs(tuliprox_session::provider_leases::STARTING_LEASE_TTL_SECS + 1)).await; + + // Queue both snapshots before either attempt can replace the expired provider tag. + let read_blocker = session.read().await; + let mut first = Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(first.as_mut()).is_pending()); + let mut second = + Box::pin(prepare_media_origin_attempt(Some(&io), &session, generation, Duration::from_secs(1))); + assert!(futures::poll!(second.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + // Let the first reserve its slot and the second wait for that slot's acquisition. + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + let first = first.await.expect("first attempt reacquires the provider lease"); + let second = second.await.expect("second attempt joins the reacquired provider lease"); + { + let session = session.read().await; + let current = session.origin_account_binding.as_ref().expect("binding"); + let joined = second.provider_lease.as_ref().expect("joined provider lease").1.binding(); + assert_ne!(current.provider_binding_tag, original_tag); + assert_eq!(joined.provider_binding_tag, current.provider_binding_tag); + assert_eq!(joined.playback_request_id, current.playback_request_id); + assert_eq!(session.activity.origin_work_generation, generation); + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 2); + assert_eq!( + session + .origin_account_io_lease + .as_ref() + .expect("shared I/O lease") + .active_io_count + .load(Ordering::Acquire), + 2 + ); + } + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(first.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 1); + assert!(second.finish(&session).await.generation_valid); + assert_eq!(io.ctx.active_provider.get_provider_connections_count(), 0); + assert_eq!(session.read().await.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert!(session.read().await.origin_account_io_lease.is_none()); + } +} diff --git a/backend/hls/src/refresh/commit.rs b/backend/hls/src/refresh/commit.rs index c9f8d117a..8b3c60d5e 100644 --- a/backend/hls/src/refresh/commit.rs +++ b/backend/hls/src/refresh/commit.rs @@ -134,6 +134,18 @@ pub(super) fn commit_fetched_manifest_with_acceptance_mode( staged_switch: Option<&HlsStagedSwitchCommit>, ) -> Result<(HlsManifestCommitProgressEvidence, bool, bool), HlsManifestCommitError> { ensure_refresh_origin_work_generation_is_current(session, request)?; + if !session + .origin_control + .qualification + .pending_is_current(&crate::short_manifest::HlsOriginQualificationBasis::capture(session)) + || (matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Checking { .. } + ) && session.origin_control.qualification.generation() + != Some(session.origin_control.acceptance_generation)) + { + return Err(switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated)); + } let existing_transient_reason = match &session.mode { HlsSessionMode::TransientPassthrough { reason } => Some(reason.clone()), HlsSessionMode::NormalCacheTimeline => None, @@ -178,6 +190,7 @@ pub(super) fn commit_fetched_manifest_with_acceptance_mode( if result.is_ok() { if matches!(session.mode, HlsSessionMode::TransientPassthrough { .. }) { session.startup = None; + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; } clear_satisfied_fresh_manifest_requirement(session, request); } @@ -254,6 +267,8 @@ fn commit_normal_fetched_manifest( )?; update_origin_provider_session_headers(session, fetched); mark_manifest_acceptance_success(session, fetched, &quality, fetch_finished_at_ms); + super::short_manifest::record_origin_qualification(session, manifest.segments.len()); + queue_and_render_normal_manifest(session, request, fetch_finished_at_ms); Ok((HlsManifestCommitProgressEvidence::CacheTimeline(refresh_timing), true, true)) } @@ -431,9 +446,23 @@ fn evaluate_manifest_acceptance_for_commit( match evaluate_manifest_acceptance(session, fetched, timeline, &fetch_context, now_ms, mode) { HlsManifestAcceptanceDecision::Accept { quality } | HlsManifestAcceptanceDecision::AcceptHostSwitch { quality } => Ok(quality), - HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => Err(HlsManifestCommitError::RetryCurrentTarget), + HlsManifestAcceptanceDecision::RetryCurrentTarget { .. } => { + if matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Qualified { .. } + ) { + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; + } + Err(HlsManifestCommitError::RetryCurrentTarget) + } HlsManifestAcceptanceDecision::Reject { reason, .. } => { session.origin_control.path_condition = crate::origin_progress::HlsOriginPathCondition::AcceptanceConflict; + if matches!( + session.origin_control.qualification, + crate::short_manifest::HlsOriginQualification::Qualified { .. } + ) { + session.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Unchecked; + } let log_reason = HlsManifestRejectLogReason::from(reason.clone()); warn!( "HLS origin manifest rejected: session={} proxy_session={} reason={} media_sequence={} segments={}", @@ -502,6 +531,7 @@ fn mark_manifest_acceptance_success( quality: &HlsManifestOriginQuality, observed_at_ms: u64, ) { + session.origin_account_manifest_pending = None; let binding = Url::parse(&fetched.resolved_request_url) .ok() .and_then(|request_url| HlsManifestOriginBinding::new(request_url, fetched.provider_url_index).ok()); @@ -570,6 +600,10 @@ fn prepare_transient_manifest_rewrite( let handoff_discontinuity_sequence = session .pending_handoff_discontinuity_sequence .or_else(|| input.quality.requires_handoff_discontinuity.then_some(0)); + let rewrite_options = TransientRewriteOptions { + handoff_discontinuity_sequence, + origin_account_generation: session.transient.origin_account_generation(), + }; let append_context = (handoff_discontinuity_sequence.is_none() && matches!(input.manifest_semantics.playlist_type, Some(HlsPlaylistType::Event))) .then(|| { @@ -606,16 +640,17 @@ fn prepare_transient_manifest_rewrite( incremental_rewrite = true; appended } else { - TransientManifestRewriter::rewrite( + TransientManifestRewriter::rewrite_with_options( input.body, input.final_manifest_url, &session.proxy_session_id, input.reverse_proxy_rewrite_secret, input.rendered_at_ms, input.transient_resource_ttl_ms, + rewrite_options, ) } - } else if handoff_discontinuity_sequence.is_some() { + } else { TransientManifestRewriter::rewrite_with_options( input.body, input.final_manifest_url, @@ -623,16 +658,7 @@ fn prepare_transient_manifest_rewrite( input.reverse_proxy_rewrite_secret, input.rendered_at_ms, input.transient_resource_ttl_ms, - TransientRewriteOptions { handoff_discontinuity_sequence }, - ) - } else { - TransientManifestRewriter::rewrite( - input.body, - input.final_manifest_url, - &session.proxy_session_id, - input.reverse_proxy_rewrite_secret, - input.rendered_at_ms, - input.transient_resource_ttl_ms, + rewrite_options, ) }; let next_transient_discontinuity_sequence = if handoff_discontinuity_sequence.is_some() { @@ -816,6 +842,7 @@ fn commit_normal_manifest( &request.reverse_proxy_rewrite_secret, rendered_at_ms, request.transient_resource_ttl_ms, + session.transient.origin_account_generation(), ); let manifest = &manifest; let previous_highwater = session.origin_seq_highwater; @@ -871,8 +898,6 @@ fn commit_normal_manifest( if provisioning_handoff { limit_publishable_normal_provisioning_handoff_tail(session, &request.strip, manifest.segments.len()); } - queue_and_render_normal_manifest(session, request, rendered_at_ms); - let last_segment_duration_ms = manifest.segments.last().map(|segment| segment.duration_ms); let target_duration_ms = manifest.target_duration.map(|duration| u64::from(duration) * 1_000); let progress = if staged_switch.is_some() { @@ -983,11 +1008,12 @@ pub(super) fn materialize_normal_key_resources( reverse_proxy_rewrite_secret: &[u8], now_ms: u64, ttl_ms: u64, + origin_account_generation: u64, ) -> Vec { let mut resources = Vec::new(); for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { let extension = key_resource_extension(&encryption.resolved_origin_uri); - let resource = TransientResourceRef::new( + let mut resource = TransientResourceRef::new( TransientResourceKind::Key, encryption.resolved_origin_uri.clone(), reverse_proxy_rewrite_secret, @@ -995,6 +1021,7 @@ pub(super) fn materialize_normal_key_resources( ttl_ms, Some(extension.clone()), ); + resource.scope_to_origin_account(origin_account_generation, reverse_proxy_rewrite_secret); encryption.proxy_resource_id = Some(resource.id.0.clone()); encryption.proxy_resource_extension = Some(extension); resources.push(resource); diff --git a/backend/hls/src/refresh/failure.rs b/backend/hls/src/refresh/failure.rs index ff617df49..90a396486 100644 --- a/backend/hls/src/refresh/failure.rs +++ b/backend/hls/src/refresh/failure.rs @@ -163,7 +163,7 @@ pub(super) fn apply_manifest_fetch_failure_signal( } else if signal.is_hard() { Some(HlsPostRefreshAvailabilityReason::HardManifestFailure) } else { - None + Some(HlsPostRefreshAvailabilityReason::RetryableManifestFailure) }; reason.map_or(HlsPostRefreshAvailabilityAction::None, |reason| HlsPostRefreshAvailabilityAction::Reevaluate { reason, diff --git a/backend/hls/src/refresh/mod.rs b/backend/hls/src/refresh/mod.rs index 17c5c0e7b..c76c3a2ff 100644 --- a/backend/hls/src/refresh/mod.rs +++ b/backend/hls/src/refresh/mod.rs @@ -50,6 +50,7 @@ use tuliprox_core::model::{AppConfig, HlsManifestRecoveryBurstConfig, ReversePro mod commit; mod failure; +mod short_manifest; mod switch_staging; mod timing; @@ -84,6 +85,8 @@ pub struct OriginRefreshState { pub last_success_at_ms: Option, pub last_error_at_ms: Option, pub in_flight: bool, + /// Ownership token for the currently running refresh, independent of work generations and timestamps. + pub refresh_identity: Option>, } impl OriginRefreshState { @@ -92,6 +95,7 @@ impl OriginRefreshState { pub fn mark_started(&mut self, now_ms: u64) { self.last_fetch_started_at_ms = Some(now_ms); self.in_flight = true; + self.refresh_identity = Some(Arc::new(())); } fn mark_success_with_timing( @@ -115,6 +119,7 @@ impl OriginRefreshState { self.last_error_at_ms = None; self.consecutive_failures = 0; self.in_flight = false; + self.refresh_identity = None; self.next_fetch_allowed_at_ms = fetch_started_at_ms.saturating_add(refresh_interval_ms); refresh_interval_ms } @@ -125,12 +130,14 @@ impl OriginRefreshState { let next_retry_delay_ms = self.next_failure_backoff_ms(); self.consecutive_failures = self.consecutive_failures.saturating_add(1); self.in_flight = false; + self.refresh_identity = None; self.next_fetch_allowed_at_ms = failed_at_ms.saturating_add(next_retry_delay_ms); } fn mark_invalidated(&mut self, finished_at_ms: u64) { self.last_fetch_finished_at_ms = Some(finished_at_ms); self.in_flight = false; + self.refresh_identity = None; } fn next_failure_backoff_ms(&self) -> u64 { @@ -167,6 +174,8 @@ pub struct OriginRefreshRequest { pub transient_resource_ttl_ms: u64, pub manifest_commit_requirement: HlsManifestCommitRequirement, pub fresh_manifest_requirement_generation: Option, + /// Filled at refresh start; a completion may only change the state owned by this token. + pub refresh_identity: Option>, pub acceptance_directive: HlsManifestAcceptanceDirective, pub access_lease_id: Option, pub now_ms: u64, @@ -183,6 +192,7 @@ pub struct HlsPostRefreshRuntime { pub enum HlsPostRefreshAvailabilityReason { DeterministicTimelineConflict, HardManifestFailure, + RetryableManifestFailure, } impl HlsPostRefreshAvailabilityReason { @@ -190,6 +200,7 @@ impl HlsPostRefreshAvailabilityReason { match self { Self::DeterministicTimelineConflict => "deterministic_timeline_conflict", Self::HardManifestFailure => "hard_manifest_failure", + Self::RetryableManifestFailure => "retryable_manifest_failure", } } } @@ -326,6 +337,34 @@ async fn mark_origin_refresh_started_with_outcome( request: &mut OriginRefreshRequest, fetch_started_at_ms: u64, ) -> HlsOriginRefreshTriggerOutcome { + if let Some(io) = request.origin_io.as_ref() { + let needs_prepare = { + let session = request.session.read().await; + io.prepared_account.as_ref().is_none_or(|snapshot| !snapshot.matches_session(&session)) + || session.origin_account_binding.as_ref().is_some_and(|binding| { + binding.is_detached() + || crate::origin::hls_origin_account_status(&io.ctx, binding) + != crate::origin::HlsOriginAccountStatus::Known + }) + }; + if let Some(runtime) = io.account_runtime.as_ref().filter(|_| needs_prepare) { + // A replacement result must not inherit an allocation prepared for the old entry. + drop(io.take_preacquired_provider_handle()); + let class = + if io.allow_grace { HlsOriginWorkClass::ManifestInteractive } else { HlsOriginWorkClass::Background }; + let Ok(prepared) = runtime + .prepare(&request.session, crate::origin_account_runtime::HlsOriginAccountWork::Manifest, class) + .await + else { + return HlsOriginRefreshTriggerOutcome::SessionUnavailable; + }; + request.origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone(); + if let Some(io) = request.origin_io.as_mut() { + io.prepared_account = Some(prepared.snapshot); + } + request.origin_entry = prepared.entry; + } + } let metrics = Arc::clone(request.segment_worker_pool.metrics()); if let Some(guard) = request.acceptance_directive.recovery_pressure_guard.clone() { let hls_proxy = Arc::clone(&request.hls_proxy); @@ -358,7 +397,13 @@ fn mark_origin_refresh_started_in_session( fetch_started_at_ms: u64, metrics: &super::HlsCacheMetrics, ) -> HlsOriginRefreshTriggerOutcome { - if session.is_gc_marked_for_removal() { + if session.is_gc_marked_for_removal() + || request + .origin_io + .as_ref() + .and_then(|io| io.prepared_account.as_ref()) + .is_some_and(|snapshot| !snapshot.matches_session(session)) + { return HlsOriginRefreshTriggerOutcome::SessionUnavailable; } if session.origin_refresh.in_flight { @@ -381,7 +426,8 @@ fn mark_origin_refresh_started_in_session( return HlsOriginRefreshTriggerOutcome::InFlight; } if fetch_started_at_ms < session.origin_refresh.next_fetch_allowed_at_ms - && request.manifest_commit_requirement.fresh_reason().is_none() + && (request.manifest_commit_requirement.fresh_reason().is_none() + || session.origin_refresh.consecutive_failures > 0) { metrics.record_refresh_skipped(); let wait_ms = session.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(fetch_started_at_ms); @@ -395,6 +441,7 @@ fn mark_origin_refresh_started_in_session( retry_at_ms: session.origin_refresh.next_fetch_allowed_at_ms, }; } + short_manifest::prepare_qualification_refresh(request, session, fetch_started_at_ms); if let Some(origin_io) = request.origin_io.as_mut() { origin_io.started_generation = Some(session.start_origin_work()); } @@ -403,6 +450,7 @@ fn mark_origin_refresh_started_in_session( .fresh_reason() .and_then(|reason| session.fresh_manifest_commit_requirement_generation(reason)); session.origin_refresh.mark_started(fetch_started_at_ms); + request.refresh_identity = session.origin_refresh.refresh_identity.clone(); metrics.record_refresh_started(); info!( "HLS origin manifest refresh started: session={} proxy_session={}", @@ -496,7 +544,26 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ committed_evidence_session, ) = { let mut session = request.session.write().await; + let owns_refresh = match (&request.refresh_identity, &session.origin_refresh.refresh_identity) { + (Some(expected), Some(current)) => Arc::ptr_eq(expected, current), + (None, _) => true, + (Some(_), None) => false, + }; match result { + // A handoff can reset refresh state and start a successor in the same millisecond. + Ok(_) | Err(_) if !owns_refresh => { + metrics.record_refresh_skipped(); + (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) + } + // Invalidated work must release its own singleflight flag even when no successor was published. + Ok(_) | Err(_) + if !origin_work_state.generation_valid + || !refresh_origin_work_generation_is_current(&session, &request) => + { + session.origin_refresh.mark_invalidated(fetch_finished_at_ms); + metrics.record_refresh_skipped(); + (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) + } Ok(CommittedOriginManifest { fetched, progress_evidence, wake_segment_scheduler, wake_map_scheduler }) => { let pending_manifest_follow_up = Some((session.proxy_session_id.clone(), session.target_duration)); let prepared_terminal_target_duration_ms = if early_prepared_terminal_target_duration_ms.is_none() { @@ -559,21 +626,8 @@ async fn refresh_and_commit(mut request: OriginRefreshRequest, fetch_started_at_ record_superseded_manifest_recovery_completion(&mut session, metrics.as_ref(), fetch_finished_at_ms); (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) } - Err(err) - if !origin_work_state.generation_valid - || !refresh_origin_work_generation_is_current(&session, &request) => - { - session.origin_refresh.mark_invalidated(fetch_finished_at_ms); - metrics.record_refresh_skipped(); - debug!( - "HLS origin manifest refresh completion discarded: session={} proxy_session={} reason=origin-work-generation-invalidated error={}", - safe_session_key(&session.key), - safe_proxy_session_id(&session.proxy_session_id), - err.log_label() - ); - (false, false, None, None, HlsPostRefreshAvailabilityAction::None, None) - } Err(err) => { + session.origin_control.qualification.record_failure(); session.origin_refresh.mark_failure(fetch_finished_at_ms); metrics.record_refresh_failed(); let post_refresh_action = apply_manifest_fetch_failure_signal(&mut session, &err, fetch_finished_at_ms); @@ -864,6 +918,7 @@ fn manifest_fetch_context(request: &OriginRefreshRequest) -> HlsOriginManifestFe HlsOriginManifestFetchContext { app_config: Arc::clone(&request.app_config), session: Arc::clone(&request.session), + availability_runtime: request.post_refresh_runtime.as_ref().map(|runtime| runtime.ctx.clone()), origin_entry: request.origin_entry.clone(), headers: hls_origin_headers_with_provider_session(&request.headers, &request.origin_provider_session_headers), client: request.client.clone(), @@ -880,7 +935,7 @@ fn manifest_fetch_context(request: &OriginRefreshRequest) -> HlsOriginManifestFe async fn fetch_and_commit_manifest_with_policy( request: &mut OriginRefreshRequest, ) -> Result { - let (established_recovery_binding, prefetch_trigger) = { + let (mut established_recovery_binding, prefetch_trigger) = { let session = request.session.read().await; if !refresh_origin_work_generation_is_current(&session, request) { return Err(OriginManifestFetchError::RetryExhausted); @@ -898,6 +953,9 @@ async fn fetch_and_commit_manifest_with_policy( (session.established_manifest_recovery_binding(), trigger) }; let fetch_context = manifest_fetch_context(request); + if let Some(result) = short_manifest::resume_qualification(request, &fetch_context).await { + return result; + } let mut recovery_suppression_logged = false; if prefetch_trigger.starts_episode() { if let Some(binding) = established_recovery_binding.clone() { @@ -944,6 +1002,20 @@ async fn fetch_and_commit_manifest_with_policy( } Err(initial_error) => return Err(initial_error), }; + // A handoff first resolves the new entry. Recovery may use only that concrete target, + // while acceptance still compares against the retained timeline and pinned host. + { + let mut session = request.session.write().await; + if session.origin_account_manifest_pending.is_some() { + if !refresh_origin_work_generation_is_current(&session, request) { + return Err(OriginManifestFetchError::RetryExhausted); + } + session.origin_control.manifest_origin_binding = url::Url::parse(&fetched.resolved_request_url) + .ok() + .and_then(|url| HlsManifestOriginBinding::new(url, fetched.provider_url_index).ok()); + established_recovery_binding = session.established_manifest_recovery_binding(); + } + } Box::pin(commit_initial_fetched_manifest( request, &fetch_context, @@ -1057,6 +1129,9 @@ async fn commit_initial_fetched_manifest( established_recovery_binding: Option, recovery_suppression_logged: bool, ) -> Result { + if let Some(result) = short_manifest::qualify_short_manifest_if_needed(request, fetch_context, &fetched).await { + return result; + } let acceptance_mode = request.manifest_commit_requirement.acceptance_mode(); let selected_report = score_hls_manifest_candidate_for_selection_log(fetch_context, &fetched, acceptance_mode).await; diff --git a/backend/hls/src/refresh/short_manifest.rs b/backend/hls/src/refresh/short_manifest.rs new file mode 100644 index 000000000..51a97738f --- /dev/null +++ b/backend/hls/src/refresh/short_manifest.rs @@ -0,0 +1,215 @@ +//! Startup qualification shares recovery selection, singleflight and retry receipts. + +use super::{commit_manifest_recovery_candidate, CommittedOriginManifest, OriginRefreshRequest}; +use crate::{ + hls_origin_log_value, + manifest_acceptance::HlsManifestAcceptanceTrigger, + manifest_fetch::{ + retry_hls_origin_manifest_recovery_chain, FetchedOriginManifest, HlsOriginManifestFetchContext, + OriginManifestFetchError, + }, + manifest_origin_binding::HlsManifestOriginBinding, + safe_proxy_session_id, safe_session_key, + short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis, HlsQualifiedOriginWindows}, + HlsFreshManifestRequiredReason, HlsSession, +}; +use tuliprox_parser::hls::origin_manifest::{ + parse_manifest_semantics, parse_origin_media_manifest, OriginManifestParseOutcome, +}; +use url::Url; + +pub(super) fn prepare_qualification_refresh(request: &mut OriginRefreshRequest, session: &mut HlsSession, now_ms: u64) { + let basis = HlsOriginQualificationBasis::capture(session); + if !session.origin_control.qualification.pending_is_current(&basis) { + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } + if !session.origin_is_qualified() { + return; + } + let Some(reason) = request.manifest_commit_requirement.fresh_reason() else { + return; + }; + let pending_requirement = session.fresh_manifest_commit_requirement_generation(reason).is_some(); + let already_qualified = !pending_requirement + && match reason { + HlsFreshManifestRequiredReason::ColdStart + | HlsFreshManifestRequiredReason::PreviousHardManifestFailure + | HlsFreshManifestRequiredReason::StaleEntryWindow => true, + HlsFreshManifestRequiredReason::ProvisioningHandoff => { + session.pending_handoff_discontinuity_sequence.is_none() + } + HlsFreshManifestRequiredReason::ExpiredRevalidation => { + // Each qualification may still be waiting for its media, + // including revalidation of an already published session. + session.origin_qualification_awaits_publication() + || crate::manifest_commit::has_servable_committed_manifest(session, now_ms) + } + }; + if already_qualified { + // Another lease completed the foundation check. Freshness remains a + // normal refresh requirement, without spending another configured burst. + request.manifest_commit_requirement = super::HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + } else if matches!( + reason, + HlsFreshManifestRequiredReason::ExpiredRevalidation + | HlsFreshManifestRequiredReason::PreviousHardManifestFailure + | HlsFreshManifestRequiredReason::ProvisioningHandoff + ) { + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } +} + +pub(super) async fn qualify_short_manifest_if_needed( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, + fetched: &FetchedOriginManifest, +) -> Option> { + if parse_manifest_semantics(&fetched.body).window_policy().preserves_full_manifest() { + return None; + } + let OriginManifestParseOutcome::Normal(manifest) = + parse_origin_media_manifest(&fetched.body, &fetched.final_manifest_url) + else { + return None; + }; + if !(1..3).contains(&manifest.segments.len()) { + return None; + } + let binding = Url::parse(&fetched.resolved_request_url) + .ok() + .and_then(|url| HlsManifestOriginBinding::new(url, fetched.provider_url_index).ok())?; + let qualified = { + let session = request.session.read().await; + session.origin_is_qualified() + && session.origin_control.manifest_origin_binding.as_ref() == Some(&binding) + && crate::manifest_fetch::fetched_effective_manifest_host(fetched).as_deref() + == session.origin_control.pinned_host.as_deref() + }; + if qualified { + return None; + } + let mut qualification_context = context.clone(); + if !fetched.provider_session_headers.is_empty() { + let mut session = request.session.write().await; + if !super::commit::refresh_origin_work_generation_is_current(&session, request) { + return Some(Err(OriginManifestFetchError::RecoveryUnavailable { + reason: crate::manifest_fetch::HlsManifestRecoveryUnavailableReason::BindingSuperseded, + })); + } + // HTTP session cookies belong to the transport. Keeping this successful + // response's cookie does not commit its media or qualify its short window. + session.origin_provider_session_headers = crate::headers::merge_hls_provider_session_headers( + &session.origin_provider_session_headers, + &fetched.provider_session_headers, + ); + qualification_context.headers = + crate::hls_origin_headers_with_provider_session(&context.headers, &session.origin_provider_session_headers); + } + let result = run_qualification(request, &qualification_context, binding).await.map(|mut committed| { + committed.fetched.attempts = committed.fetched.attempts.saturating_add(fetched.attempts.saturating_sub(1)); + committed.fetched.candidate_requests = + committed.fetched.candidate_requests.saturating_add(fetched.candidate_requests); + committed + }); + Some(result) +} + +pub(super) async fn resume_qualification( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, +) -> Option> { + let target = { + let mut session = request.session.write().await; + let target = session.origin_qualification_retry_target().cloned()?; + if let Some(reason) = request.manifest_commit_requirement.fresh_reason() { + let trigger = request.manifest_commit_requirement.strict_handoff_trigger(); + if trigger.starts_episode() + && session.fresh_manifest_commit_requirement_generation(reason).is_some() + && session + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.trigger() != trigger) + { + // A new lifecycle finding gets one new full check of the known + // concrete target. Further leases inherit that check's receipt. + session.origin_control.qualification = HlsOriginQualification::Unchecked; + } + } + target + }; + Some(run_qualification(request, context, target).await) +} + +async fn run_qualification( + request: &OriginRefreshRequest, + context: &HlsOriginManifestFetchContext, + binding: HlsManifestOriginBinding, +) -> Result { + let trigger = { + let session = request.session.read().await; + session + .origin_qualification_retry_target() + .and_then(|_| { + session + .origin_control + .acceptance_episode + .as_ref() + .map(crate::manifest_acceptance::HlsManifestAcceptanceEpisode::trigger) + }) + .unwrap_or_else(|| { + let trigger = super::manifest_recovery_trigger(request); + if trigger.starts_episode() { + trigger + } else { + HlsManifestAcceptanceTrigger::Observe + } + }) + }; + Box::pin(retry_hls_origin_manifest_recovery_chain( + context, + binding, + None, + None, + trigger, + request.manifest_commit_requirement.acceptance_mode(), + |fetched, mode| Box::pin(commit_manifest_recovery_candidate(request, fetched, mode)), + )) + .await +} + +pub(super) fn record_origin_qualification(session: &mut HlsSession, segment_count: usize) { + let basis = HlsOriginQualificationBasis::capture(session); + if session.origin_control.qualification.qualified_windows(&basis).is_some() { + return; + } + let generation = session.origin_control.acceptance_generation; + let full_burst = session.origin_control.qualification.generation() == Some(generation) + && matches!(session.origin_control.qualification, HlsOriginQualification::Checking { .. }) + && session + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.generation == generation && episode.full_burst_completed); + let windows = if (1..3).contains(&segment_count) && full_burst { + log::warn!( + "HLS short origin window qualified: RFC 8216 section 6.2.2 recommends three target durations; host={} session={} proxy_session={} segments={segment_count} qualification={} burst={}", + session.origin_control.pinned_host.as_deref().map_or_else(|| "none".to_string(), hls_origin_log_value), + safe_session_key(&session.key), safe_proxy_session_id(&session.proxy_session_id), generation.0, + session.origin_control.acceptance_episode.as_ref().map_or(0, |episode| episode.completed_burst_candidates) + ); + HlsQualifiedOriginWindows::ShortAllowed + } else if segment_count >= 3 { + HlsQualifiedOriginWindows::Regular + } else { + return; + }; + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis, + generation, + windows, + publication_boundary: session.last_normal_manifest_commit_identity(), + }; +} diff --git a/backend/hls/src/refresh/switch_staging.rs b/backend/hls/src/refresh/switch_staging.rs index 7cc21218e..9c29c0676 100644 --- a/backend/hls/src/refresh/switch_staging.rs +++ b/backend/hls/src/refresh/switch_staging.rs @@ -325,25 +325,26 @@ pub(super) async fn stage_alternative_manifest_switch( return Err(switch_staging_error(HlsManifestRejectLogReason::SwitchResourceUnavailable)); } }; - let candidate_key_resources = materialize_normal_key_resources( - &mut manifest, - &request.reverse_proxy_rewrite_secret, - request.now_ms, - request.transient_resource_ttl_ms, - ); let effective_host = fetched_effective_manifest_host(fetched) .ok_or_else(|| switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated))?; let effective_host_id = effective_origin_host_id(&effective_host); - let (generation, preview) = { + let (generation, preview, candidate_key_resources) = { let session = request.session.read().await; let generation = switch_staging_generation(&session) .ok_or_else(|| switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated))?; if !generation.candidate_identity.matches_candidate(Some(&effective_host), &fetched.body) { return Err(switch_staging_error(HlsManifestRejectLogReason::StagedSwitchInvalidated)); } + let candidate_key_resources = materialize_normal_key_resources( + &mut manifest, + &request.reverse_proxy_rewrite_secret, + request.now_ms, + request.transient_resource_ttl_ms, + session.transient.origin_account_generation(), + ); let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).map_err(handoff_preview_error)?; - (generation, preview) + (generation, preview, candidate_key_resources) }; let first_segment = preview .segments diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index a72e98143..947b13d59 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -61,12 +61,15 @@ use super::{ }; mod acceptance; +mod account_handoff; mod archive_master; mod critical_handoff; mod headers_redirects; mod origin_binding; mod ownership; +mod reduced_retry; mod retry_policy; +mod short_manifest; mod startup_representation; mod support; mod switch_staging; diff --git a/backend/hls/src/refresh/tests/acceptance.rs b/backend/hls/src/refresh/tests/acceptance.rs index 8751977c8..aa25e6355 100644 --- a/backend/hls/src/refresh/tests/acceptance.rs +++ b/backend/hls/src/refresh/tests/acceptance.rs @@ -247,6 +247,7 @@ pub(in crate::refresh::tests) fn install_stored_provisioning_manifest(session: & host_local_index: u32::try_from(*proxy_seq).unwrap_or(u32::MAX), }, proxy_seq: *proxy_seq, + media_start_ms: *proxy_seq * 2_000, duration_ms: 2_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -267,6 +268,7 @@ pub(in crate::refresh::tests) fn install_stored_provisioning_manifest(session: & session.publishable_origin_head_proxy_seq = segment_proxy_seqs.first().copied(); session.publishable_origin_tail_proxy_seq = segment_proxy_seqs.last().copied(); session.proxy_next_seq = Some(3); + session.next_media_start_ms = 6_000; session.target_duration = Some(2); assert_eq!( session.store_rendered_manifest(RenderedManifest { diff --git a/backend/hls/src/refresh/tests/account_handoff.rs b/backend/hls/src/refresh/tests/account_handoff.rs new file mode 100644 index 000000000..52223dd58 --- /dev/null +++ b/backend/hls/src/refresh/tests/account_handoff.rs @@ -0,0 +1,139 @@ +use super::{ + mark_origin_refresh_started_with_outcome, refresh_and_commit, test_origin_refresh_request, test_session, + HlsOriginRefreshTriggerOutcome, +}; +use crate::{HlsCtx, HlsOriginIoContext, HlsSessionHandle}; +use std::sync::Arc; +use tuliprox_core::model::{Config, PlaybackKind}; +use tuliprox_session::ConnectionKind; + +pub(super) fn request_with_origin_work(session: &HlsSessionHandle) -> super::OriginRefreshRequest { + let mut request = test_origin_refresh_request(Arc::clone(session)); + request.origin_io = Some(HlsOriginIoContext { + ctx: HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:1234".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: None, + }); + request +} + +#[tokio::test] +async fn invalidated_refresh_without_successful_handoff_releases_its_singleflight() { + let session = test_session(); + let mut old = request_with_origin_work(&session); + assert_eq!(mark_origin_refresh_started_with_outcome(&mut old, 100).await, HlsOriginRefreshTriggerOutcome::Started); + // A failed account replacement invalidates work but publishes no successor state. + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(old, 100).await; + { + let current = session.read().await; + assert!(!current.origin_refresh.in_flight); + assert_eq!(current.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 0); + } + let mut retry = request_with_origin_work(&session); + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut retry, 100).await, + HlsOriginRefreshTriggerOutcome::Started + ); + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(retry, 100).await; +} + +#[tokio::test] +async fn invalidated_refresh_cannot_release_a_successor_started_in_the_same_millisecond() { + let session = test_session(); + let mut old = request_with_origin_work(&session); + assert_eq!(mark_origin_refresh_started_with_outcome(&mut old, 100).await, HlsOriginRefreshTriggerOutcome::Started); + { + let mut current = session.write().await; + current.invalidate_queued_origin_work(); + current.origin_refresh = super::OriginRefreshState::default(); + } + let mut successor = request_with_origin_work(&session); + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut successor, 100).await, + HlsOriginRefreshTriggerOutcome::Started + ); + let successor_state = session.read().await.origin_refresh.clone(); + refresh_and_commit(old, 100).await; + { + let current = session.read().await; + assert!(current.origin_refresh.in_flight); + assert!(Arc::ptr_eq( + current.origin_refresh.refresh_identity.as_ref().unwrap(), + successor_state.refresh_identity.as_ref().unwrap() + )); + assert_eq!(current.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 1); + } + session.write().await.invalidate_queued_origin_work(); + refresh_and_commit(successor, 100).await; + assert!(!session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn account_handoff_preserves_cached_ciphertext_key_and_assigns_new_keys_to_uncached_media() { + let session = test_session(); + let request = test_origin_refresh_request(Arc::clone(&session)); + let fetched = super::fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\n10.ts\n#EXTINF:4,\n11.ts\n#EXTINF:4,\n12.ts\n"); + let mut current = session.write().await; + super::commit_fetched_manifest(&mut current, &fetched, &request, 100).unwrap(); + let old_key = current.segments[&0].encryption.clone().unwrap().resource_id; + current.segments.get_mut(&0).unwrap().status = + crate::SegmentCacheStatus::Ready { content_length: 16, ready_at_ms: 100 }; + let owner = current.proxy_session_id.clone(); + current.handoff_origin_account(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("new"), + owner.0, + 1, + 101, + )); + let next = super::fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:13\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\n13.ts\n#EXTINF:4,\n14.ts\n#EXTINF:4,\n15.ts\n"); + super::commit_fetched_manifest(&mut current, &next, &request, 102).unwrap(); + assert_eq!(current.segments[&0].encryption.as_ref().unwrap().resource_id, old_key); + let new_key = ¤t.segments[&3].encryption.as_ref().unwrap().resource_id; + assert_ne!(new_key, &old_key); + assert!(current.transient.resources.contains_key(&old_key)); + assert!(current.transient.resources.contains_key(new_key)); +} + +#[tokio::test] +async fn refresh_start_rejects_a_changed_binding_even_if_the_work_generation_is_unchanged() { + let session = test_session(); + let mut request = request_with_origin_work(&session); + { + let mut current = session.write().await; + let owner = current.proxy_session_id.0.clone(); + current.replace_origin_account_binding(Some(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("old"), + owner.clone(), + 1, + 100, + ))); + let snapshot = crate::origin_account_runtime::HlsOriginAccountSnapshot::from_session(¤t); + let generation = snapshot.generation; + request.origin_io.as_mut().unwrap().prepared_account = Some(snapshot); + current.replace_origin_account_binding(Some(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("new"), + owner, + 2, + 100, + ))); + assert_eq!(current.activity.origin_work_generation, generation); + } + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 100).await, + HlsOriginRefreshTriggerOutcome::SessionUnavailable + ); + assert!(!session.read().await.origin_refresh.in_flight); +} diff --git a/backend/hls/src/refresh/tests/archive_master.rs b/backend/hls/src/refresh/tests/archive_master.rs index f3bc2f2af..f720deb54 100644 --- a/backend/hls/src/refresh/tests/archive_master.rs +++ b/backend/hls/src/refresh/tests/archive_master.rs @@ -31,6 +31,7 @@ async fn archive_master_resolution_binds_recovery_to_entry_and_scrubs_redirected headers.insert(header::COOKIE, HeaderValue::from_static("sid=entry-secret")); let context = HlsOriginManifestFetchContext { app_config: test_app_config(), + availability_runtime: None, session: Arc::new(RwLock::new(HlsSession::new( HlsSessionKey::new(1, "12345").with_archive_reference(1_784_898_000), b"secret", diff --git a/backend/hls/src/refresh/tests/headers_redirects.rs b/backend/hls/src/refresh/tests/headers_redirects.rs index c1aa7e8c6..04734b975 100644 --- a/backend/hls/src/refresh/tests/headers_redirects.rs +++ b/backend/hls/src/refresh/tests/headers_redirects.rs @@ -76,6 +76,7 @@ async fn refresh_stores_headers_after_hls_origin_policy() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, now_ms: 100, @@ -218,6 +219,7 @@ async fn shared_initial_manifest_automatic_cross_origin_redirect_keeps_identity_ let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers, client: reqwest::Client::builder().no_proxy().build().expect("client"), @@ -276,6 +278,7 @@ async fn shared_initial_manifest_budget_covers_status_decoder_redirect_and_succe let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), diff --git a/backend/hls/src/refresh/tests/origin_binding.rs b/backend/hls/src/refresh/tests/origin_binding.rs index 80534fc1b..94e5dae6a 100644 --- a/backend/hls/src/refresh/tests/origin_binding.rs +++ b/backend/hls/src/refresh/tests/origin_binding.rs @@ -1,8 +1,8 @@ use super::{ assert_stale_switch_generation_rejects_commit, bind_refresh_request_to_app_state, classify_manifest_fetch_failure, commit_fetched_manifest, fetch_and_commit_manifest_with_policy, fetched_manifest, host_from_base_url, - install_published_recovery_binding, manifest_body, manifest_fetch_context, mark_origin_refresh_started, - no_delay_policy, path_has_extension, prepare_cross_host_baseline, publish_ready_test_manifest, refresh_and_commit, + install_published_recovery_binding, manifest_fetch_context, mark_origin_refresh_started, no_delay_policy, + path_has_extension, prepare_cross_host_baseline, publish_ready_test_manifest, refresh_and_commit, request_header_value, resolved_hls_manifest_request_url_from_input, retry_test_manifest_recovery_chain, spawn_test_origin, switch_manifest_body, switch_test_request, test_app_config, test_origin_refresh_request, test_segment_repair_manager, test_session, three_segment_manifest_body, trigger_origin_refresh_sync, @@ -102,7 +102,7 @@ async fn accepted_manifest_commit_stores_provider_session_cookie_separately() { #[tokio::test] async fn provider_failover_status_does_not_count_as_hls_retry() { let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; - let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { name: "demo".into(), urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], @@ -143,6 +143,7 @@ async fn provider_failover_status_does_not_count_as_hls_retry() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -183,7 +184,7 @@ async fn provider_failover_status_does_not_count_as_hls_retry() { assert!(!session.key.stable_value().contains("provider://")); assert!(!session.key.stable_value().contains(first.base_url.as_str())); assert!(!session.key.stable_value().contains(second.base_url.as_str())); - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(102)); assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); let metrics = segment_worker_pool.metrics().snapshot(); assert_eq!(metrics.refresh_started, 1); @@ -305,11 +306,7 @@ async fn provider_failover_initial_success_commits_without_hls_host_retry_when_u let second = spawn_test_origin(Arc::new(move |_path| { let hit = second_hits_for_handler.fetch_add(1, Ordering::SeqCst); if hit == 0 { - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:102\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n102.ts\n".to_string(), - ); + return (200, Vec::new(), three_segment_manifest_body(100)); } ( 200, @@ -359,6 +356,7 @@ async fn provider_failover_initial_success_commits_without_hls_host_retry_when_u transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, diff --git a/backend/hls/src/refresh/tests/reduced_retry.rs b/backend/hls/src/refresh/tests/reduced_retry.rs new file mode 100644 index 000000000..b0c4deb8d --- /dev/null +++ b/backend/hls/src/refresh/tests/reduced_retry.rs @@ -0,0 +1,338 @@ +use super::*; +use crate::{ + manifest_acceptance::{HlsManifestAcceptanceEpisode, HlsManifestAcceptanceEpisodeStatus}, + session_store::HlsSessionHandle, + SegmentCacheStatus, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::Mutex; + +const BASELINE: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/baseline.m3u8")); +const CONFLICT: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/conflict.m3u8")); +const REPAIRED: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/reduced-retry/repaired.m3u8")); + +async fn commit_baseline(request: &OriginRefreshRequest, context: &HlsOriginManifestFetchContext) { + let baseline = + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(context)).await.unwrap(); + { + let mut s = request.session.write().await; + let now_ms = tuliprox_core::utils::current_time_millis(); + commit_fetched_manifest(&mut s, &baseline, request, now_ms).unwrap(); + s.activity.last_authorized_media_at_ms = Some(now_ms); + for segment in s.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; + } + s.render_and_store_manifest(now_ms).unwrap(); + assert_eq!(s.origin_seq_highwater, Some(1005)); + assert!(s.established_manifest_recovery_binding().is_some()); + } +} + +async fn commit_observed_recovery_candidate( + request: &OriginRefreshRequest, + fetched: FetchedOriginManifest, + mode: HlsManifestCommitAcceptanceMode, + observations: &Mutex>, +) -> Result { + let before = { + let s = request.session.read().await; + let score = score_manifest_recovery_candidate(&s, &fetched, &manifest_fetch_context(request)).unwrap(); + assert_eq!(score.quality.host_relation, crate::manifest_fetch::HlsManifestOriginRelation::SameRedirectHost); + assert_eq!(score.quality.sequence_relation, HlsManifestSequenceRelation::PlausibleForward); + eprintln!( + "selected candidate host={:?} sequence={:?}", + score.quality.host_relation, score.quality.sequence_relation + ); + s.origin_control.acceptance_episode.clone().unwrap() + }; + let outcome = super::super::commit_manifest_recovery_candidate(request, fetched, mode).await; + let label = match &outcome { + Ok(_) => "committed".to_string(), + Err(HlsManifestCommitError::TimelineRejected { reason }) => reason.status_label(), + Err(error) => format!("{error:?}"), + }; + eprintln!( + "real commit callback={} generation={} full_bursts={} timing={:?}", + label, + before.generation.0, + before.full_bursts_completed, + before.timing() + ); + observations.lock().await.push((label, before)); + outcome +} + +async fn conflict_then_reduced_retry(permanent_conflict: bool) { + let recovery = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(AtomicUsize::new(0)); + let (phase, count) = (Arc::clone(&recovery), Arc::clone(&requests)); + let server = spawn_test_origin(Arc::new(move |_| { + let body = if phase.load(Ordering::Acquire) { + let n = count.fetch_add(1, Ordering::AcqRel) + 1; + eprintln!("recovery HTTP request={n} status=200 permanent={permanent_conflict}"); + if n <= 12 || permanent_conflict { + CONFLICT + } else { + REPAIRED + } + } else { + BASELINE + }; + (200, Vec::new(), body.to_string()) + })) + .await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + let url = format!("{}/index.m3u8", server.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&url).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + let context = manifest_fetch_context(&request); + commit_baseline(&request, &context).await; + recovery.store(true, Ordering::Release); + let callbacks = Arc::new(Mutex::new(Vec::new())); + let observations = Arc::clone(&callbacks); + let request_ref = &request; + let result = tokio::time::timeout( + std::time::Duration::from_secs(15), + retry_hls_origin_manifest_recovery_chain( + &context, + test_manifest_origin_binding(url::Url::parse(&url).unwrap()), + None, + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + move |fetched, mode| { + let observations = Arc::clone(&observations); + async move { commit_observed_recovery_candidate(request_ref, fetched, mode, &observations).await } + }, + ), + ) + .await + .unwrap(); + let records = callbacks.lock().await; + let s = session.read().await; + let episode = s.origin_control.acceptance_episode.as_ref().unwrap(); + eprintln!( + "recovery result={:?} requests={} callbacks={} episode={episode:?}", + result.as_ref().map(|_| ()), + requests.load(Ordering::Acquire), + records.len() + ); + assert!( + records[0].0.contains("origin-sequence-resource-conflict"), + "first commit must encounter the real conflict: {}", + records[0].0 + ); + if permanent_conflict { + assert!(result.is_err()); + assert_eq!(requests.load(Ordering::Acquire), 16); + assert_eq!(records.len(), 5, "all reduced candidates must reach the real commit guard"); + assert!(records.iter().all(|(label, _)| label.contains("origin-sequence-resource-conflict"))); + assert!(matches!( + crate::manifest_acceptance::manifest_acceptance_episode_status( + Some(episode), + episode.generation, + request.now_ms + ), + HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } + )); + assert_eq!(s.origin_seq_highwater, Some(1005)); + } else { + assert!( + result.is_ok(), + "repaired single-candidate retry must commit; requests={} callbacks={}", + requests.load(Ordering::Acquire), + records.len() + ); + assert_eq!(requests.load(Ordering::Acquire), 13); + assert_eq!(records.len(), 2); + assert_eq!(records[1].0, "committed"); + assert_eq!(s.origin_seq_highwater, Some(1008)); + } + assert_eq!(episode.full_bursts_completed, 1); + for (_, before) in records.iter() { + assert_eq!(before.generation, episode.generation); + assert_eq!(before.timing(), episode.timing()); + assert_eq!(before.full_bursts_completed, 1); + } +} + +#[tokio::test] +async fn reduced_retry_commits_repaired_same_host_manifest_after_real_resource_conflict() { + conflict_then_reduced_retry(false).await; +} + +#[tokio::test] +async fn reduced_retry_permanent_resource_conflict_exhausts_existing_budget() { + conflict_then_reduced_retry(true).await; +} + +async fn replace_acceptance_episode_with_successor( + session: &HlsSessionHandle, + retry_binding: HlsManifestOriginBinding, + account_handoff: bool, + after_selection: bool, +) -> HlsManifestAcceptanceEpisode { + use crate::short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis}; + + let mut s = session.write().await; + let old = s.origin_control.acceptance_episode.as_ref().unwrap().clone(); + if after_selection { + assert!(old.exhaustion_reason().is_none()); + assert_eq!(old.state, crate::manifest_acceptance::HlsManifestAcceptanceState::Committing); + } else { + assert!(old.exhaustion_reason().is_some()); + } + if account_handoff { + let owner = s.proxy_session_id.0.clone(); + s.handoff_origin_account(crate::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("account-b"), + owner, + 1, + 200, + )); + } else { + s.invalidate_queued_origin_work(); + } + let now = tuliprox_core::utils::current_time_millis(); + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let generation = s.origin_control.begin_acceptance_episode( + now, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(now, plan), + ); + assert_ne!(generation, old.generation); + s.origin_control.qualification = HlsOriginQualification::Checking { + basis: HlsOriginQualificationBasis::capture(&s), + generation, + target: retry_binding, + }; + s.origin_control.acceptance_episode.clone().unwrap() +} + +async fn superseded_reduced_response_preserves_successor(account_handoff: bool, after_selection: bool) { + use super::support::{spawn_test_origin_with_response_gate, TestOriginResponseGate}; + use tokio::sync::Notify; + + let requests = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&requests); + let gate = Arc::new(TestOriginResponseGate { + request_number: 14, // Baseline plus twelve burst samples, then the reduced retry. + entered: Notify::new(), + release: Notify::new(), + }); + let server = spawn_test_origin_with_response_gate( + Arc::new(move |_| { + let n = count.fetch_add(1, Ordering::AcqRel); + let body = if n == 0 { + BASELINE + } else if n <= 12 { + CONFLICT + } else { + REPAIRED + }; + (200, Vec::new(), body.to_string()) + }), + if after_selection { None } else { Some(Arc::clone(&gate)) }, + ) + .await; + let session = test_session(); + let mut request = super::account_handoff::request_with_origin_work(&session); + request.origin_io.as_mut().unwrap().started_generation = Some(session.read().await.activity.origin_work_generation); + let url = format!("{}/index.m3u8", server.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&url).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast; + let context = manifest_fetch_context(&request); + commit_baseline(&request, &context).await; + let baseline = session.read().await.origin_to_proxy.clone(); + let callbacks = Arc::new(AtomicUsize::new(0)); + let observed_callbacks = Arc::clone(&callbacks); + let binding = test_manifest_origin_binding(url::Url::parse(&url).unwrap()); + let retry_binding = binding.clone(); + let commit_gate = Arc::clone(&gate); + let recovery = tokio::spawn(async move { + retry_hls_origin_manifest_recovery_chain( + &context, + binding, + None, + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, mode| { + let number = observed_callbacks.fetch_add(1, Ordering::AcqRel) + 1; + let gate = Arc::clone(&commit_gate); + let request_ref = &request; + async move { + if after_selection && number == 2 { + gate.entered.notify_one(); + gate.release.notified().await; + } + super::super::commit_manifest_recovery_candidate(request_ref, fetched, mode).await + } + }, + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), gate.entered.notified()).await.unwrap(); + let successor = + replace_acceptance_episode_with_successor(&session, retry_binding, account_handoff, after_selection).await; + gate.release.notify_one(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), recovery).await.unwrap().unwrap(); + { + let s = session.read().await; + let actual = s.origin_control.acceptance_episode.as_ref().unwrap(); + eprintln!("REVIEW: after_selection={after_selection} handoff={account_handoff} result={:?} callbacks={} requests={} expected_generation={} actual_generation={} expected_state={:?} actual_state={:?} successor_unchanged={}", + result.as_ref().map(|_| ()), callbacks.load(Ordering::Acquire), requests.load(Ordering::Acquire), + successor.generation.0, actual.generation.0, successor.state, actual.state, actual == &successor); + assert!(actual == &successor, "old commit must leave the successor episode unchanged"); + } + assert!(matches!( + result, + Err(OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded + }) + )); + assert_eq!( + callbacks.load(Ordering::Acquire), + if after_selection { 2 } else { 1 }, + "old response must not reach another commit" + ); + assert_eq!(requests.load(Ordering::Acquire), 14, "old chain stops before another attempt"); + let s = session.read().await; + assert_eq!(s.origin_to_proxy, baseline); + assert_eq!( + s.origin_control.acceptance_episode.as_ref(), + Some(&successor), + "old work must leave the pending successor unchanged" + ); + assert_eq!(s.origin_control.qualification.generation(), Some(successor.generation)); +} + +#[tokio::test] +async fn reduced_retry_held_response_cannot_mutate_successor_after_account_handoff() { + superseded_reduced_response_preserves_successor(true, false).await; +} + +#[tokio::test] +async fn reduced_retry_held_response_cannot_mutate_successor_after_work_generation_change() { + superseded_reduced_response_preserves_successor(false, false).await; +} + +#[tokio::test] +async fn review_selected_commit_cannot_mutate_successor_after_account_handoff() { + superseded_reduced_response_preserves_successor(true, true).await; +} + +#[tokio::test] +async fn review_selected_commit_cannot_mutate_successor_after_work_generation_change() { + superseded_reduced_response_preserves_successor(false, true).await; +} diff --git a/backend/hls/src/refresh/tests/retry_policy.rs b/backend/hls/src/refresh/tests/retry_policy.rs index 24e7ce718..5319497c2 100644 --- a/backend/hls/src/refresh/tests/retry_policy.rs +++ b/backend/hls/src/refresh/tests/retry_policy.rs @@ -761,6 +761,7 @@ async fn concurrent_maybe_trigger_origin_refresh_starts_singleflight_once() { transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -1081,7 +1082,7 @@ async fn compatible_aes_128_manifest_commits_normal_timeline_with_opaque_key_res #[tokio::test] async fn cold_recovery_directive_is_suppressed_until_baseline() { - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(0)))).await; let session = test_session(); let mut request = test_origin_refresh_request(Arc::clone(&session)); request.origin_entry = @@ -1097,7 +1098,7 @@ async fn cold_recovery_directive_is_suppressed_until_baseline() { assert_eq!(manifest_requests, 1); assert_ne!(manifest_requests, plan.total_candidates()); let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(2)); assert!(session.origin_control.manifest_origin_binding.is_some()); assert!(session.origin_control.acceptance_episode.is_none()); } @@ -1118,6 +1119,7 @@ async fn shared_initial_manifest_decoder_failure_retries_until_success() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1164,6 +1166,7 @@ async fn shared_initial_manifest_waits_for_next_attempt_base_delay() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1202,6 +1205,7 @@ async fn shared_initial_manifest_decoder_failures_stop_at_attempt_budget() { let context = HlsOriginManifestFetchContext { app_config: test_app_config(), session: test_session(), + availability_runtime: None, origin_entry, headers: HeaderMap::new(), client: reqwest::Client::new(), @@ -1266,7 +1270,7 @@ async fn shared_refresh_metrics_use_successful_manifest_attempt_count() { .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) .cloned() .collect::>(); - assert_eq!(manifest_requests.len(), 3); + assert_eq!(manifest_requests.len(), 4, "initial retries plus the configured single-candidate qualification"); assert!(manifest_requests .iter() .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); @@ -1400,7 +1404,7 @@ async fn recovery_unavailable_after_valid_response_preserves_response_evidence() #[tokio::test] async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_recovery() { let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(0)))).await; let session = test_session(); { let mut session = session.write().await; @@ -1421,7 +1425,7 @@ async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_r assert_eq!(manifest_requests, 1); assert_ne!(manifest_requests, plan.total_candidates()); let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.origin_seq_highwater, Some(2)); assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); assert!(session.origin_control.acceptance_episode.is_none()); } @@ -1449,3 +1453,38 @@ async fn cold_start_hard_error_is_preserved_by_fetch_policy() { super::super::super::origin_progress::HlsOriginProgressPhase::Cold ); } + +#[tokio::test] +async fn shared_initial_manifest_caps_retry_after_at_half_target_duration() { + for use_manual_redirects in [false, true] { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { + return (502, vec![("Retry-After", "60".to_string())], "retry".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let session = test_session(); + session.write().await.origin_control.record_media_progress(50, 1_000); + let mut request = test_origin_refresh_request(session); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.use_manual_redirects = use_manual_redirects; + let context = super::manifest_fetch_context(&request); + let started_at = std::time::Instant::now(); + + let fetched = tokio::time::timeout( + std::time::Duration::from_secs(5), + fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)), + ) + .await + .expect("Retry-After must not stall the manifest fetch for 60 seconds") + .expect("second logical attempt should succeed"); + + assert_eq!(fetched.attempts, 2); + assert_eq!(server.requests.lock().await.len(), 2); + assert!(started_at.elapsed() >= std::time::Duration::from_millis(500)); + } +} diff --git a/backend/hls/src/refresh/tests/short_manifest.rs b/backend/hls/src/refresh/tests/short_manifest.rs new file mode 100644 index 000000000..7353bc17f --- /dev/null +++ b/backend/hls/src/refresh/tests/short_manifest.rs @@ -0,0 +1,669 @@ +use super::*; +use crate::{ + short_manifest::HlsOriginQualification, HlsAccessLeaseId, HlsFreshManifestRequiredReason, HlsManifestRenderer, + SegmentCacheStatus, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::{ + fmt::Write as _, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +use tuliprox_core::utils::current_time_millis; + +fn playlist(head: u64, count: usize) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + for offset in 0..count { + writeln!(body, "#EXTINF:2.5,\nsegment-{}.ts", head + u64::try_from(offset).unwrap()).unwrap(); + } + body +} + +fn request_for_origin(server: &TestOriginServer) -> OriginRefreshRequest { + let mut request = test_origin_refresh_request(test_session()); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).unwrap(); + request.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly; + request.now_ms = current_time_millis(); + request +} + +async fn configure_first_ready(request: &OriginRefreshRequest) { + let config = tuliprox_core::model::HlsStartupConfig { + mode: shared::model::HlsStartupMode::FirstReady, + ..Default::default() + }; + request.session.write().await.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 10_000, + worker: Arc::downgrade(&request.segment_worker_pool), + access_leases: Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())), + revisions: std::collections::BTreeMap::new(), + policy_fixed: true, + store: Arc::new(crate::SegmentRevisionStore::default()), + budget: crate::ProgressiveBudgetManager::new(config), + }); +} + +fn publish_first_ready_head(session: &mut crate::HlsSession, now_ms: u64) { + let head = session.publishable_origin_head_proxy_seq.unwrap(); + let startup = session.startup.as_mut().unwrap(); + let revision = + startup.store.create(session.proxy_session_id.clone(), head, crate::SegmentRevisionKind::Processed).unwrap(); + assert!(revision + .revision() + .complete(crate::CachedSegmentMetadata { path: "first-ready-test.ts".into(), size: 128 })); + startup.revisions.insert(head, revision); + session.segments.get_mut(&head).unwrap().status = + SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: now_ms }; + session.render_and_store_manifest(now_ms).unwrap(); +} + +#[tokio::test] +async fn short_startup_one_segment_waits_for_complete_burst() { startup_with_short_window(1).await; } + +#[tokio::test] +async fn short_startup_two_segments_waits_for_complete_burst() { startup_with_short_window(2).await; } + +async fn startup_with_short_window(count: usize) { + let server = spawn_test_origin(Arc::new(move |_| (200, Vec::new(), playlist(100, count)))).await; + let mut request = request_for_origin(&server); + request.strip.value = 10; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert!(session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 1 + request.manifest_recovery_burst.level.plan().total_candidates()); + assert_eq!(session.origin_control.qualification.generation(), Some(session.origin_control.acceptance_generation)); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + let rendered = session.render_and_store_manifest(request.now_ms).unwrap(); + assert_eq!(rendered.segment_proxy_seqs.len(), count); + assert_eq!(rendered.playlist_duration_ms, u64::try_from(count).unwrap() * 2_500); + assert_eq!(session.initial_prefetch_gap_segments, 0); + assert!(!rendered.body.contains("#EXT-X-ENDLIST")); +} + +#[tokio::test] +async fn short_qualification_survives_ten_refreshes_and_larger_intermediate_windows() { + for short_count in [1, 2] { + let head = Arc::new(AtomicUsize::new(100)); + let count = Arc::new(AtomicUsize::new(short_count)); + let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count)); + let server = spawn_test_origin(Arc::new(move |_| { + ( + 200, + Vec::new(), + playlist( + u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), + origin_count.load(Ordering::SeqCst), + ), + ) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + let startup_requests = server.requests.lock().await.len(); + for cycle in 1..=10 { + head.store(100 + cycle * 4, Ordering::SeqCst); + count.store(if cycle % 2 == 1 { short_count + 2 } else { short_count }, Ordering::SeqCst); + { + let mut session = request.session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = 0; + } + request.now_ms = current_time_millis(); + request.access_lease_id = Some(HlsAccessLeaseId(format!("new-lease-{cycle}"))); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(session.allows_short_origin_windows()); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + let rendered = session.render_and_store_manifest(request.now_ms).unwrap(); + assert!(rendered.segment_proxy_seqs.len() >= count.load(Ordering::SeqCst)); + assert_eq!(server.requests.lock().await.len(), startup_requests + cycle); + } + } +} + +async fn concurrent_refreshes(request: &OriginRefreshRequest, leases: usize) -> usize { + let mut tasks = tokio::task::JoinSet::new(); + for lease in 0..leases { + let mut request = request.clone(); + request.access_lease_id = Some(HlsAccessLeaseId(format!("lease-{lease}"))); + tasks.spawn(trigger_origin_refresh_sync(request)); + } + let mut started = 0; + while let Some(result) = tasks.join_next().await { + started += usize::from(result.unwrap()); + } + started +} + +#[tokio::test] +async fn twenty_leases_share_startup_revalidation_and_normal_freshness_requests() { + let mut request_counts = Vec::new(); + for leases in [1, 20] { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + let initial_count = server.requests.lock().await.len(); + let first_generation = request.session.read().await.origin_control.acceptance_generation; + { + let mut session = request.session.write().await; + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + } + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + let session = request.session.read().await; + assert!(session.allows_short_origin_windows(), "{:?} {:?}", session.origin_control, session.origin_refresh); + assert!(session.origin_control.acceptance_generation.0 > first_generation.0); + assert!(session.fresh_manifest_commit_required.is_none()); + drop(session); + head.store(102, Ordering::SeqCst); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::StaleEntryWindow, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, leases).await, 1); + request_counts.push((initial_count, server.requests.lock().await.len())); + } + assert_eq!(request_counts[0], request_counts[1]); +} + +#[tokio::test] +async fn short_failed_qualification_preserves_full_burst_receipt_and_backoff_for_new_leases() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + (200, Vec::new(), playlist(100, 1)) + } else { + (500, Vec::new(), String::new()) + } + })) + .await; + let mut request = request_for_origin(&server); + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + assert!(matches!( + request.session.read().await.origin_control.qualification, + HlsOriginQualification::RetryPending { .. } + )); + // The existing failure schedule allows an immediate first retry, then backs off. + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + let before_retry = calls.load(Ordering::SeqCst); + assert!(trigger_origin_refresh_sync(request.clone()).await); + assert_eq!( + calls.load(Ordering::SeqCst), + before_retry + 1, + "failed qualification retries one sample, never a new full burst" + ); + let failed_state = request.session.read().await.origin_refresh.clone(); + request.now_ms = failed_state.next_fetch_allowed_at_ms - 1; + assert_eq!(concurrent_refreshes(&request, 20).await, 0); + let session = request.session.read().await; + assert_eq!(session.origin_refresh, failed_state); + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(!session.allows_short_origin_windows()); + assert!(session.segments.is_empty()); +} + +#[tokio::test] +async fn later_regular_candidate_wins_before_short_startup_is_published() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + let count = if origin_calls.fetch_add(1, Ordering::SeqCst) < 2 { 1 } else { 3 }; + (200, Vec::new(), playlist(100, count)) + })) + .await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.segments.len(), 3); + assert!(!session.allows_short_origin_windows()); + assert_eq!(calls.load(Ordering::SeqCst), 3); +} + +#[tokio::test] +async fn unqualified_short_window_stays_unrenderable_and_unchanged_refresh_does_not_advance_media() { + let session = test_session(); + let request = test_origin_refresh_request(Arc::clone(&session)); + let short = fetched_manifest(&playlist(100, 1)); + let mut session = session.write().await; + commit_fetched_manifest(&mut session, &short, &request, 100).unwrap(); + assert!(!session.allows_short_origin_windows()); + assert!(HlsManifestRenderer::render(&session, 100).is_err()); + let progress = session.origin_control.progress_generation; + commit_fetched_manifest(&mut session, &short, &request, 200).unwrap(); + assert_eq!(session.origin_control.progress_generation, progress); + assert_eq!(session.origin_control.last_media_progress_at_ms, Some(100)); +} + +#[tokio::test] +async fn short_qualification_rejects_stale_epoch_binding_and_acceptance_generation_before_commit() { + use crate::short_manifest::{HlsOriginQualification, HlsOriginQualificationBasis}; + for change in 0..3 { + let handle = test_session(); + let request = test_origin_refresh_request(Arc::clone(&handle)); + let plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + let mut session = handle.write().await; + let generation = session.origin_control.begin_acceptance_episode( + 100, + plan, + HlsManifestAcceptanceTrigger::Observe, + &test_acceptance_episode_timing(100, plan), + ); + let target = test_manifest_origin_binding(url::Url::parse("https://origin.example/live.m3u8").unwrap()); + session.origin_control.qualification = HlsOriginQualification::Checking { + basis: HlsOriginQualificationBasis::capture(&session), + generation, + target: target.clone(), + }; + session.origin_control.acceptance_episode.as_mut().unwrap().record_full_burst(); + match change { + 0 => session.origin_epoch += 1, + 1 => session.origin_control.manifest_origin_binding = Some(target), + 2 => session.origin_control.acceptance_generation.0 += 1, + _ => unreachable!(), + } + let result = commit_fetched_manifest(&mut session, &fetched_manifest(&playlist(100, 1)), &request, 200); + assert!(matches!( + result, + Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::StagedSwitchInvalidated, + }) + )); + assert!(!session.allows_short_origin_windows()); + assert!(session.segments.is_empty()); + assert_eq!(session.origin_manifest_commits(), 0); + } +} + +struct QualificationLogger(std::sync::Mutex>); + +impl log::Log for QualificationLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { metadata.level() == log::Level::Warn } + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + let message = record.args().to_string(); + if message.starts_with("HLS short origin window qualified:") { + self.0.lock().unwrap().push(message); + } + } + } + fn flush(&self) {} +} + +#[tokio::test] +async fn short_window_warning_occurs_once_per_qualification_across_new_leases() { + static LOGGER: QualificationLogger = QualificationLogger(std::sync::Mutex::new(Vec::new())); + log::set_logger(&LOGGER).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + request.session = Arc::new(tokio::sync::RwLock::new(crate::HlsSession::new( + crate::HlsSessionKey::new(1, "warning-once"), + b"secret", + current_time_millis(), + ))); + assert!(trigger_origin_refresh_sync(request.clone()).await); + for lease in 0..10 { + request.access_lease_id = Some(HlsAccessLeaseId(format!("warning-lease-{lease}"))); + request.now_ms = current_time_millis(); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + assert!(trigger_origin_refresh_sync(request.clone()).await); + } + let proxy_session = crate::safe_proxy_session_id(&request.session.read().await.proxy_session_id); + let warnings = || { + LOGGER + .0 + .lock() + .unwrap() + .iter() + .filter(|message| message.contains(&format!("proxy_session={proxy_session}"))) + .cloned() + .collect::>() + }; + let first = warnings(); + assert_eq!(first.len(), 1); + assert!(first[0].contains("RFC 8216 section 6.2.2")); + assert!(first[0].contains(&format!("host={}", crate::hls_origin_log_value("127.0.0.1")))); + assert!(first[0].contains("segments=1 qualification=1 burst=2")); + head.store(101, Ordering::SeqCst); + request.session.write().await.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request).await); + assert_eq!(warnings().len(), 2); +} + +#[tokio::test] +async fn short_permission_does_not_release_empty_replayed_or_failed_media() { + let server = spawn_test_origin(Arc::new(|_| (200, Vec::new(), playlist(100, 1)))).await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let mut session = request.session.write().await; + assert!(session.allows_short_origin_windows()); + let proxy_seq = session.publishable_origin_head_proxy_seq.unwrap(); + session.segments.get_mut(&proxy_seq).unwrap().status = SegmentCacheStatus::FailedPermanent { + failed_at_ms: request.now_ms, + status: Some(axum::http::StatusCode::NOT_FOUND), + }; + assert!(HlsManifestRenderer::render(&session, request.now_ms).is_err()); + session.segments.get_mut(&proxy_seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + session.render_and_store_manifest(request.now_ms).unwrap(); + let commits = session.origin_manifest_commits(); + let mut replay = fetched_manifest(&playlist(101, 1).replace("segment-101.ts", "segment-100.ts")); + replay.final_manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); + replay.resolved_request_url.clone_from(&replay.final_manifest_url); + replay.redirect_host = None; + assert!(matches!( + commit_fetched_manifest(&mut session, &replay, &request, request.now_ms + 1), + Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::PublishedResourceReplay { .. } + }) + )); + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_manifest_commits(), commits); + let mut empty = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:102\n"); + empty.final_manifest_url = replay.final_manifest_url; + empty.resolved_request_url = replay.resolved_request_url; + empty.redirect_host = None; + assert!(commit_fetched_manifest(&mut session, &empty, &request, request.now_ms + 2).is_err()); + assert_eq!(session.origin_manifest_commits(), commits); +} + +#[tokio::test] +async fn expired_manifest_requires_new_qualification_without_an_explicit_fresh_requirement_flag() { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + { + let mut session = request.session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + session.render_and_store_manifest(request.now_ms).unwrap(); + assert!(session.last_normal_manifest_commit_identity().is_some()); + session.last_rendered_manifest = None; + } + assert!(request.session.read().await.fresh_manifest_commit_required.is_none()); + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert!(session.origin_control.acceptance_generation.0 > generation.0); + assert!(session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 6); +} + +#[tokio::test] +async fn recovered_origin_requalifies_after_expired_episode_under_shared_backoff() { + let phase = Arc::new(AtomicUsize::new(0)); + let origin_phase = Arc::clone(&phase); + let server = spawn_test_origin(Arc::new(move |_| match origin_phase.load(Ordering::SeqCst) { + 0 => (200, Vec::new(), playlist(100, 1)), + 1 => (500, Vec::new(), String::new()), + _ => (200, Vec::new(), playlist(101, 1)), + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + { + let mut session = request.session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 128, ready_at_ms: request.now_ms }; + } + session.render_and_store_manifest(request.now_ms).unwrap(); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::ExpiredRevalidation); + } + phase.store(1, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let failed_generation = { + let mut session = request.session.write().await; + let previous = session.origin_control.acceptance_episode.take().unwrap(); + let mut expired = crate::manifest_acceptance::HlsManifestAcceptanceEpisode::new( + previous.generation, + 1, + previous.burst_plan, + previous.trigger(), + &test_acceptance_episode_timing(1, previous.burst_plan), + ); + expired.record_full_burst(); + expired.state = previous.state; + expired.observed_landscape = previous.observed_landscape; + assert!(expired.timing().acceptance_deadline.as_millis_since_epoch() < current_time_millis()); + session.origin_control.acceptance_episode = Some(expired); + assert_eq!(session.origin_seq_highwater, Some(100)); + session.origin_control.acceptance_generation + }; + let requests_before_retry = server.requests.lock().await.len(); + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, failed_generation); + let retry_at_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + request.now_ms = retry_at_ms - 1; + phase.store(2, Ordering::SeqCst); + assert_eq!(concurrent_refreshes(&request, 20).await, 0); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 1); + request.now_ms = retry_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + let session = request.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(101)); + assert_eq!(session.origin_control.acceptance_generation.0, failed_generation.0 + 1); + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_refresh.consecutive_failures, 0); + assert!(session.fresh_manifest_commit_required.is_none()); + assert_eq!(server.requests.lock().await.len(), requests_before_retry + 4); +} + +#[tokio::test] +async fn new_leases_reuse_origin_qualification_before_first_publication() { + let server = spawn_test_origin(Arc::new(|_| (200, Vec::new(), playlist(100, 1)))).await; + let mut request = request_for_origin(&server); + configure_first_ready(&request).await; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = { + let session = request.session.read().await; + assert!(session.origin_is_qualified()); + assert!(session.last_rendered_manifest.is_none()); + assert!(session.last_normal_manifest_commit_identity().is_none()); + assert!(session.segments.values().all(|segment| !matches!(segment.status, SegmentCacheStatus::Ready { .. }))); + session.origin_control.acceptance_generation + }; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, generation); + assert_eq!(server.requests.lock().await.len(), 4); +} + +#[tokio::test] +async fn first_ready_revalidation_is_shared_until_its_publication_then_can_expire_again() { + let head = Arc::new(AtomicUsize::new(100)); + let origin_head = Arc::clone(&head); + let server = spawn_test_origin(Arc::new(move |_| { + (200, Vec::new(), playlist(u64::try_from(origin_head.load(Ordering::SeqCst)).unwrap(), 1)) + })) + .await; + let mut request = request_for_origin(&server); + configure_first_ready(&request).await; + assert!(trigger_origin_refresh_sync(request.clone()).await); + let previous_commit = { + let mut session = request.session.write().await; + publish_first_ready_head(&mut session, request.now_ms); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + assert!(!crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + session.last_normal_manifest_commit_identity().unwrap() + }; + head.store(101, Ordering::SeqCst); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + { + let session = request.session.read().await; + assert_eq!(session.startup_mode(), shared::model::HlsStartupMode::FirstReady); + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert_eq!(session.origin_seq_highwater, Some(101)); + assert!(session.origin_is_qualified()); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + assert!(!crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + } + assert_eq!(server.requests.lock().await.len(), 5); + request.now_ms = current_time_millis(); + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + { + let mut session = request.session.write().await; + assert_eq!(session.origin_control.acceptance_generation.0, 2); + assert!(session.origin_is_qualified()); + assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit)); + assert_eq!(server.requests.lock().await.len(), 6); + publish_first_ready_head(&mut session, request.now_ms); + assert!(session.last_normal_manifest_commit_identity().unwrap() > previous_commit); + assert!(crate::manifest_commit::has_servable_committed_manifest(&session, request.now_ms)); + session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0; + } + head.store(102, Ordering::SeqCst); + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.origin_control.acceptance_generation.0, 3); + assert_eq!(session.origin_seq_highwater, Some(102)); + assert!(session.origin_is_qualified()); + assert_eq!(server.requests.lock().await.len(), 8); +} + +#[tokio::test] +async fn genuine_lifecycle_revalidation_replaces_failed_startup_once_and_shares_its_failure() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + (200, Vec::new(), playlist(100, 1)) + } else { + (500, Vec::new(), String::new()) + } + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + request + .session + .write() + .await + .require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + }; + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + let revalidated = request.session.read().await.origin_control.acceptance_generation; + assert!(revalidated.0 > generation.0); + let requests_after_revalidation = calls.load(Ordering::SeqCst); + request.now_ms = request.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert_eq!(concurrent_refreshes(&request, 20).await, 1); + assert_eq!(calls.load(Ordering::SeqCst), requests_after_revalidation + 1); + assert_eq!(request.session.read().await.origin_control.acceptance_generation, revalidated); +} + +#[tokio::test] +async fn segment_count_change_preserves_regular_qualification_without_starting_a_new_burst() { + let count = Arc::new(AtomicUsize::new(3)); + let origin_count = Arc::clone(&count); + let server = spawn_test_origin(Arc::new(move |_| { + let count = origin_count.load(Ordering::SeqCst); + (200, Vec::new(), playlist(if count == 3 { 100 } else { 104 }, count)) + })) + .await; + let mut request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let generation = request.session.read().await.origin_control.acceptance_generation; + assert!(request.session.read().await.origin_is_qualified()); + assert!(!request.session.read().await.allows_short_origin_windows()); + count.store(1, Ordering::SeqCst); + request.session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::StaleEntryWindow }; + request.now_ms = current_time_millis(); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let session = request.session.read().await; + assert_eq!(session.origin_control.acceptance_generation, generation); + assert!(session.origin_is_qualified()); + assert!(!session.allows_short_origin_windows()); + assert_eq!(server.requests.lock().await.len(), 2); +} + +#[tokio::test] +async fn short_startup_preserves_first_response_provider_cookie_for_burst_and_media_fetches() { + let calls = Arc::new(AtomicUsize::new(0)); + let origin_calls = Arc::clone(&calls); + let server = spawn_test_origin(Arc::new(move |_| { + let headers = if origin_calls.fetch_add(1, Ordering::SeqCst) == 0 { + vec![("Set-Cookie", "sid=short-origin; Path=/; HttpOnly".to_string())] + } else { + Vec::new() + }; + (200, headers, playlist(100, 1)) + })) + .await; + let request = request_for_origin(&server); + assert!(trigger_origin_refresh_sync(request.clone()).await); + let raw_requests = server.raw_requests.lock().await; + assert_eq!(raw_requests.len(), 3); + assert!(raw_requests[1..] + .iter() + .all(|request| request_header_value(request, "cookie") == Some("sid=short-origin"))); + let session = request.session.read().await; + assert!(session.allows_short_origin_windows()); + assert_eq!(session.origin_provider_session_headers.get(axum::http::header::COOKIE).unwrap(), "sid=short-origin"); + assert!(!session.origin_request_headers.contains_key(axum::http::header::COOKIE)); +} diff --git a/backend/hls/src/refresh/tests/support.rs b/backend/hls/src/refresh/tests/support.rs index 8a1dcaef4..56521feb3 100644 --- a/backend/hls/src/refresh/tests/support.rs +++ b/backend/hls/src/refresh/tests/support.rs @@ -169,6 +169,7 @@ pub(in crate::refresh::tests) async fn refresh_session_with_origin_body(body: &' transient_resource_ttl_ms: 300_000, manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, fresh_manifest_requirement_generation: None, + refresh_identity: None, acceptance_directive: HlsManifestAcceptanceDirective::none(), access_lease_id: None, disabled_headers: None, @@ -206,6 +207,19 @@ impl Drop for TestOriginServer { } pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHandler) -> TestOriginServer { + spawn_test_origin_with_response_gate(handler, None).await +} + +pub(in crate::refresh::tests) struct TestOriginResponseGate { + pub request_number: usize, + pub entered: Notify, + pub release: Notify, +} + +pub(in crate::refresh::tests) async fn spawn_test_origin_with_response_gate( + handler: TestOriginHandler, + gate: Option>, +) -> TestOriginServer { let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); let addr = listener.local_addr().expect("local addr"); let requests = Arc::new(Mutex::new(Vec::new())); @@ -220,6 +234,7 @@ pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHand let requests = Arc::clone(&requests_for_task); let raw_requests = Arc::clone(&raw_requests_for_task); let handler = Arc::clone(&handler); + let gate = gate.clone(); tokio::spawn(async move { let mut buf = vec![0_u8; 4096]; let mut used = 0_usize; @@ -241,9 +256,17 @@ pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHand let request = String::from_utf8_lossy(&buf[..used]).into_owned(); let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/").to_string(); - requests.lock().await.push(path.clone()); + let request_number = { + let mut requests = requests.lock().await; + requests.push(path.clone()); + requests.len() + }; raw_requests.lock().await.push(request); let (status, headers, body) = handler(path); + if let Some(gate) = gate.filter(|gate| gate.request_number == request_number) { + gate.entered.notify_one(); + gate.release.notified().await; + } let reason = match status { 200 => "OK", 302 => "Found", @@ -351,6 +374,7 @@ pub(in crate::refresh::tests) fn test_origin_refresh_request(session: Arc bool { fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: RenderTailPolicy) -> Option> { let head_seq = session.publishable_origin_head_proxy_seq?; + let minimum_visible_segments = if session.allows_short_origin_windows() { 1 } else { MIN_VISIBLE_SEGMENTS }; if let Some(startup) = &session.startup { // The head always needs body bytes; only optimistic successors may still be waiting for theirs. if !startup.revisions.get(&head_seq).is_some_and(|revision| revision.revision().is_publishable()) { @@ -272,7 +273,7 @@ fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: } window.push(seq); } - if window.len() < MIN_VISIBLE_SEGMENTS { + if window.len() < minimum_visible_segments { return None; } // Fast-start state lives as long as the session, so live slides render here too. @@ -332,7 +333,7 @@ fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: } retain_published_predecessors(session, &mut window); - if window.len() < MIN_VISIBLE_SEGMENTS { + if window.len() < minimum_visible_segments { return None; } // Lease admission enforces the READY-duration reserve. The renderer only diff --git a/backend/hls/src/response/tests.rs b/backend/hls/src/response/tests.rs index 12852e7d0..38a23ec1a 100644 --- a/backend/hls/src/response/tests.rs +++ b/backend/hls/src/response/tests.rs @@ -100,6 +100,7 @@ impl CachedSegmentFixture { host_local_index: u32::try_from(proxy_seq).expect("test proxy sequence"), }, proxy_seq, + media_start_ms: proxy_seq * 4_000, duration_ms: 4_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/runtime_custom_tail.rs b/backend/hls/src/runtime_custom_tail.rs index 77e9120e6..d04509367 100644 --- a/backend/hls/src/runtime_custom_tail.rs +++ b/backend/hls/src/runtime_custom_tail.rs @@ -1,7 +1,7 @@ use super::{ availability::{ commit_prepared_runtime_custom_tail, commit_terminal_tail_if_lease_reserve_requires_cutover, - HlsTerminalResolution, + terminal_bundle_failure_compatibility, terminal_bundle_incompatibility, HlsTerminalResolution, }, hls_ctx::HlsCtx, lease::{new_hls_access_lease_id, HlsRuntimePolicyRevocation, HlsRuntimePolicyRevocationOutcome}, @@ -12,7 +12,8 @@ use super::{ session_store::HlsSessionHandle, terminal_tail::{ snapshot_terminal_media_asset, terminal_media_asset_identity, HlsLeasePlaybackMode, HlsTerminalAssetIdentity, - HlsTerminalMediaAsset, HlsTerminalTailCompatibility, HLS_TERMINAL_TAIL_SEGMENT_COUNT, + HlsTerminalMediaAsset, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, }, HlsAccessLeaseId, ProxySessionId, }; @@ -359,7 +360,9 @@ async fn resolve_hls_runtime_custom_tail_after_revocation( return HlsRuntimeCustomTailOutcome::NoLiveLeaseContext; }; match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => return HlsRuntimeCustomTailOutcome::AlreadyCommitted, + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => { + return HlsRuntimeCustomTailOutcome::AlreadyCommitted + } HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { return HlsRuntimeCustomTailOutcome::Superseded; } @@ -580,6 +583,7 @@ impl HlsStandaloneCustomAccessStore { username: Arc::clone(&entry.access.username), shared_lease: entry.access.shared_lease.clone(), bytes, + terminal_generation: None, }) } @@ -605,6 +609,7 @@ pub struct HlsStandaloneCustomSegmentAccess { pub username: Arc, pub shared_lease: Option, pub bytes: bytes::Bytes, + pub(crate) terminal_generation: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -620,52 +625,92 @@ pub fn standalone_custom_asset_fingerprint(identity: HlsTerminalAssetIdentity) - /// Resolves one immutable prepared standalone segment without scanning or /// rewriting MPEG-TS at the HTTP request boundary. -pub fn resolve_hls_standalone_custom_segment( +pub async fn resolve_hls_standalone_custom_segment( ctx: &HlsCtx, lease_id: &HlsAccessLeaseId, asset_fingerprint: &str, index: u16, now_ms: u64, ) -> Result { - ctx.hls_proxy.resolve_standalone_custom_segment(lease_id, asset_fingerprint, index, now_ms) + ctx.hls_proxy.resolve_standalone_custom_segment(lease_id, asset_fingerprint, index, now_ms).await } -async fn await_prepared_standalone_bundle( +pub(crate) async fn await_prepared_standalone_bundle( ctx: &HlsCtx, asset: Arc, key: HlsPreparedTerminalBundleKey, -) -> Option> { +) -> Result, HlsTerminalTailCompatibility> { let state = ctx.hls_proxy.prepared_terminal_bundle_state(key).unwrap_or_else(|| { ctx.hls_proxy.start_prepared_terminal_bundle(asset, key.target_duration_ms, key.segment_count) }); - match state { - HlsPreparedTerminalBundleState::Ready { bundle } if bundle.matches_key_and_shape(key) => Some(bundle), - HlsPreparedTerminalBundleState::Preparing { .. } => { - let HlsPreparedTerminalBundleObservation::Flight(ticket) = - ctx.hls_proxy.observe_prepared_terminal_bundle(key) - else { - return None; - }; - let timeout_ms = ctx.hls_proxy.origin_manifest_timeout_ms().max(1); - match tokio::time::timeout(Duration::from_millis(timeout_ms), ticket.wait()).await { - Ok(HlsPreparedTerminalBundleCompletion::Ready { bundle }) if bundle.matches_key_and_shape(key) => { - Some(bundle) + let state = match state { + HlsPreparedTerminalBundleState::Preparing { key: preparing_key } if preparing_key == key => { + match ctx.hls_proxy.observe_prepared_terminal_bundle(key) { + HlsPreparedTerminalBundleObservation::Flight(ticket) => { + let timeout_ms = ctx.hls_proxy.origin_manifest_timeout_ms().max(1); + let completion = tokio::time::timeout(Duration::from_millis(timeout_ms), ticket.wait()) + .await + .map_err(|_| HlsTerminalTailCompatibility::TerminalMediaNotReady)?; + return prepared_standalone_completion_result(completion, key); + } + HlsPreparedTerminalBundleObservation::Settled(state) => state, + HlsPreparedTerminalBundleObservation::Missing => { + return Err(HlsTerminalTailCompatibility::TerminalMediaNotReady) } - Ok( - HlsPreparedTerminalBundleCompletion::Ready { .. } - | HlsPreparedTerminalBundleCompletion::Failed { .. } - | HlsPreparedTerminalBundleCompletion::Incompatible { .. } - | HlsPreparedTerminalBundleCompletion::FlightReplaced { .. }, - ) - | Err(_) => None, } } - HlsPreparedTerminalBundleState::Ready { .. } - | HlsPreparedTerminalBundleState::Failed { .. } - | HlsPreparedTerminalBundleState::Incompatible { .. } => None, + state => state, + }; + prepared_standalone_bundle_result(state, key) +} + +// Both cached states and completed flights retain the same key-bound compatibility diagnosis. +fn prepared_standalone_bundle_result( + state: HlsPreparedTerminalBundleState, + key: HlsPreparedTerminalBundleKey, +) -> Result, HlsTerminalTailCompatibility> { + match state { + HlsPreparedTerminalBundleState::Ready { bundle } if bundle.matches_key_and_shape(key) => Ok(bundle), + HlsPreparedTerminalBundleState::Ready { .. } => Err(HlsTerminalTailCompatibility::AssetRevisionMismatch), + HlsPreparedTerminalBundleState::Failed { key: actual_key, reason } => { + Err(terminal_bundle_failure_compatibility(actual_key, key, reason)) + } + HlsPreparedTerminalBundleState::Incompatible { key: actual_key, reason } => Err(if actual_key == key { + terminal_bundle_incompatibility(reason) + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }), + HlsPreparedTerminalBundleState::Preparing { key: actual_key } => Err(if actual_key == key { + HlsTerminalTailCompatibility::TerminalMediaNotReady + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }), } } +fn prepared_standalone_completion_result( + completion: HlsPreparedTerminalBundleCompletion, + key: HlsPreparedTerminalBundleKey, +) -> Result, HlsTerminalTailCompatibility> { + let state = match completion { + HlsPreparedTerminalBundleCompletion::Ready { bundle } => HlsPreparedTerminalBundleState::Ready { bundle }, + HlsPreparedTerminalBundleCompletion::Failed { key, reason } => { + HlsPreparedTerminalBundleState::Failed { key, reason } + } + HlsPreparedTerminalBundleCompletion::Incompatible { key, reason } => { + HlsPreparedTerminalBundleState::Incompatible { key, reason } + } + HlsPreparedTerminalBundleCompletion::FlightReplaced { key: actual_key, generation } => { + return Err(if actual_key == key && generation > 0 { + HlsTerminalTailCompatibility::TerminalMediaNotReady + } else { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }) + } + }; + prepared_standalone_bundle_result(state, key) +} + pub async fn build_hls_standalone_custom_plan( ctx: &HlsCtx, base_url: &str, @@ -681,12 +726,9 @@ pub async fn build_hls_standalone_custom_plan( target_duration_ms, segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, }; - let Some(prepared_bundle) = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), prepared_key).await - else { - return Err(HlsRuntimeCustomTailCompatibility::Incompatible( - HlsTerminalTailCompatibility::TerminalMediaNotReady, - )); - }; + let prepared_bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), prepared_key) + .await + .map_err(HlsRuntimeCustomTailCompatibility::Incompatible)?; let key = HlsStandaloneCustomPlanKey { reason, asset: media, segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT }; let asset_fingerprint = standalone_custom_asset_fingerprint(media); let extinf = format_hls_duration_ms(snapshot.asset.duration_ms()); @@ -871,4 +913,78 @@ mod tests { ); assert!(store.resolve(&HlsAccessLeaseId("lease-1".to_string()), "0000000000000001", 0, 1,).is_ok()); } + #[test] + fn standalone_preparation_preserves_cached_and_completed_incompatibility() { + use crate::prepared_terminal_bundle::{ + HlsPreparedTerminalBundleBuildError, HlsPreparedTerminalBundleFailure, + HlsPreparedTerminalBundleIncompatibility, + }; + let key = standalone_access_entry("preparation-error", 7, 100).plan.prepared_bundle.key; + let reason = + HlsPreparedTerminalBundleIncompatibility::TargetDurationExceeded { asset_ms: 12_000, target_ms: 1_000 }; + let expected = HlsTerminalTailCompatibility::TargetDurationExceeded { asset_ms: 12_000, target_ms: 1_000 }; + for state in [ + HlsPreparedTerminalBundleState::Incompatible { key, reason }, + HlsPreparedTerminalBundleState::Failed { + key, + reason: HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::Incompatible( + reason, + )), + }, + ] { + assert_eq!(prepared_standalone_bundle_result(state, key).unwrap_err(), expected); + } + for completion in [ + HlsPreparedTerminalBundleCompletion::Incompatible { key, reason }, + HlsPreparedTerminalBundleCompletion::Failed { + key, + reason: HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::Incompatible( + reason, + )), + }, + ] { + assert_eq!(prepared_standalone_completion_result(completion, key).unwrap_err(), expected); + } + } + + #[test] + fn standalone_preparation_preserves_invalid_asset_and_rejects_stale_keys() { + use crate::prepared_terminal_bundle::{ + HlsPreparedTerminalBundleBuildError, HlsPreparedTerminalBundleFailure, + HlsPreparedTerminalBundleIncompatibility, + }; + let key = standalone_access_entry("invalid-preparation", 7, 100).plan.prepared_bundle.key; + let invalid = + HlsPreparedTerminalBundleFailure::Build(HlsPreparedTerminalBundleBuildError::PublishedBundleShapeMismatch); + assert_eq!( + prepared_standalone_completion_result( + HlsPreparedTerminalBundleCompletion::Failed { key, reason: invalid }, + key + ) + .unwrap_err(), + HlsTerminalTailCompatibility::InvalidAsset + ); + let stale_key = HlsPreparedTerminalBundleKey { target_duration_ms: 2_000, ..key }; + for completion in [ + HlsPreparedTerminalBundleCompletion::Failed { key: stale_key, reason: invalid }, + HlsPreparedTerminalBundleCompletion::Incompatible { + key: stale_key, + reason: HlsPreparedTerminalBundleIncompatibility::ZeroTargetDuration, + }, + HlsPreparedTerminalBundleCompletion::FlightReplaced { key: stale_key, generation: 1 }, + ] { + assert_eq!( + prepared_standalone_completion_result(completion, key).unwrap_err(), + HlsTerminalTailCompatibility::AssetRevisionMismatch + ); + } + assert_eq!( + prepared_standalone_completion_result( + HlsPreparedTerminalBundleCompletion::FlightReplaced { key, generation: 1 }, + key + ) + .unwrap_err(), + HlsTerminalTailCompatibility::TerminalMediaNotReady + ); + } } diff --git a/backend/hls/src/segment_fetcher.rs b/backend/hls/src/segment_fetcher.rs index c4c12a54f..26df1a71f 100644 --- a/backend/hls/src/segment_fetcher.rs +++ b/backend/hls/src/segment_fetcher.rs @@ -1,20 +1,19 @@ #![allow(clippy::large_futures)] use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, classify_hls_backpressure, - fetch_hls_transient_origin_response_with_attempt_prepare, finish_hls_origin_account_io, hls_object_body_deadline, + build_hls_origin_resource_headers, classify_hls_backpressure, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_object_body_deadline, run_hls_origin_resource_retry_loop_with_attempt_prepare, transient::transient_object_expires_at, - CachedSegmentMetadata, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, - HlsBoundAccountAcquireErrorKind, HlsCacheCapacityRevision, HlsCacheMetrics, HlsCacheObjectKey, - HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, HlsOriginResourceBodyDeadline, - HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, - HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentEncryption, HlsSegmentFailureObject, - HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, - HlsSegmentRepairSource, HlsSessionHandle, HlsTransientObjectFetchFinalizer, HlsTransientOriginFetchRequest, - OriginSegmentFetchRef, OriginSegmentKey, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, - SegmentFetchPriority, StagedCacheObject, TransientObjectFetchDecision, TransientObjectFetchToken, - TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, - TransientResourceKind, TransientResourceRef, + CachedSegmentMetadata, HlsAccessLeaseId, HlsAccessLeaseStore, HlsBackpressureState, HlsCacheCapacityRevision, + HlsCacheMetrics, HlsCacheObjectKey, HlsOriginByteRangeExpectation, HlsOriginIoContext, + HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentCache, HlsSegmentEncryption, + HlsSegmentFailureObject, HlsSegmentFailureTransition, HlsSegmentFile, HlsSegmentRepairManager, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, HlsSessionHandle, HlsTransientObjectFetchFinalizer, + HlsTransientOriginFetchRequest, OriginSegmentFetchRef, OriginSegmentKey, ProxySessionId, SegmentCacheKey, + SegmentCacheStatus, SegmentEntry, SegmentFetchPriority, StagedCacheObject, TransientObjectFetchDecision, + TransientObjectFetchToken, TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, + TransientResourceId, TransientResourceKind, TransientResourceRef, }; use arc_swap::ArcSwap; use axum::http::HeaderMap; @@ -91,6 +90,18 @@ pub struct HlsSegmentWorkerPool { access_leases: Arc>, metrics: Arc, availability_reevaluations: Option>, + #[cfg(any(test, feature = "test-support"))] + origin_prepare_barrier: std::sync::Mutex>>, +} + +/// One-shot observation points around scheduler account preparation, before any queue is consumed. +#[cfg(any(test, feature = "test-support"))] +#[derive(Default)] +pub struct HlsSegmentPrepareBarrier { + pub entered: tokio::sync::Notify, + pub prepare: tokio::sync::Notify, + pub prepared: tokio::sync::Notify, + pub proceed: tokio::sync::Notify, } #[cfg(test)] @@ -104,6 +115,7 @@ mod origin; mod policy; mod scheduler; mod startup_fill; +mod startup_write; mod switch_staging; #[cfg(test)] diff --git a/backend/hls/src/segment_fetcher/commit.rs b/backend/hls/src/segment_fetcher/commit.rs index e7c076cdb..9564cfe87 100644 --- a/backend/hls/src/segment_fetcher/commit.rs +++ b/backend/hls/src/segment_fetcher/commit.rs @@ -19,11 +19,7 @@ pub(super) struct SegmentFetchCommit { pub(super) generation_valid: bool, } -#[derive(Clone, Copy)] -pub(super) struct SegmentOriginWorkFinish { - pub(super) generation_valid: bool, - pub(super) refresh_reservation: bool, -} +pub(super) use crate::origin_attempt::HlsMediaOriginWorkFinish as SegmentOriginWorkFinish; pub(super) fn segment_fetch_attempt_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { entry.origin_key == snapshot.origin_key diff --git a/backend/hls/src/segment_fetcher/demand.rs b/backend/hls/src/segment_fetcher/demand.rs index cef185b08..fdc27a052 100644 --- a/backend/hls/src/segment_fetcher/demand.rs +++ b/backend/hls/src/segment_fetcher/demand.rs @@ -125,6 +125,27 @@ impl HlsSegmentWorkerPool { { return None; } + let needs_key_io = { + let session = context.session.read().await; + session.segments.values().filter_map(|entry| entry.encryption.as_ref()).any(|encryption| { + session + .transient + .ready_key_object_valid_until_ms( + &proxy_session_id, + &encryption.resource_id, + &encryption.resource_extension, + now_ms, + ) + .is_none() + }) + }; + if needs_key_io + && crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err() + { + return None; + } let mut session = context.session.write().await; if session.is_gc_marked_for_removal() || session.active_segment_fetches >= policy.max_session_segment_fetches { return None; @@ -186,7 +207,9 @@ impl HlsSegmentWorkerPool { } } drop(permit); - if result.is_ok() { + if result.is_ok() + || context.session.read().await.origin_handoff_ready_after(snapshot.binding.origin_work_generation) + { self.schedule_wake(context, current_time_millis()); } } diff --git a/backend/hls/src/segment_fetcher/key_dependency.rs b/backend/hls/src/segment_fetcher/key_dependency.rs index 18fce0043..f84a6a5b2 100644 --- a/backend/hls/src/segment_fetcher/key_dependency.rs +++ b/backend/hls/src/segment_fetcher/key_dependency.rs @@ -80,6 +80,19 @@ pub(super) fn select_key_dependency( { return SegmentKeyDependencySelection::Unavailable; } + if !session.transient.origin_resource_is_fetchable(&resource, now_ms) + && session + .transient + .ready_key_object_valid_until_ms( + proxy_session_id, + &encryption.resource_id, + &encryption.resource_extension, + now_ms, + ) + .is_none() + { + return SegmentKeyDependencySelection::Unavailable; + } let resource_file = TransientResourceFile { resource_id: encryption.resource_id.clone(), extension: encryption.resource_extension.clone(), @@ -190,8 +203,15 @@ pub(super) async fn fetch_segment_key_dependency_into_cache( token.clone(), 1_000, ); - let (staged, origin_work) = - stage_segment_key_dependency(context, policy, &token, &resource, &resource_file).await?; + let (staged, origin_work) = stage_segment_key_dependency( + context, + policy, + &token, + &resource, + &resource_file, + binding.origin_work_generation, + ) + .await?; let generation_valid = origin_work.generation_valid && { let session = context.session.read().await; segment_key_dependency_generation_matches( @@ -249,11 +269,12 @@ async fn stage_segment_key_dependency( token: &TransientObjectFetchToken, resource: &TransientResourceRef, resource_file: &TransientResourceFile, + expected_generation: u64, ) -> Result<(StagedCacheObject, SegmentOriginWorkFinish), SegmentFetchError> { let request = HlsTransientOriginFetchRequest { resolved_origin_uri: resource.resolved_origin_uri.clone(), origin_headers: context.headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), + origin_provider_session_headers: context.session.read().await.origin_provider_session_headers.clone(), range_header: None, resource_file: resource_file.clone(), resource_kind: TransientResourceKind::Key, @@ -268,12 +289,22 @@ async fn stage_segment_key_dependency( super::super::HlsLogIdentity::from_session(&session) }, }; + let resource_for_prepare = resource.clone(); let prepare_context = context.clone(); let prepare_policy = policy.clone(); let response = fetch_hls_transient_origin_response_with_attempt_prepare(request, move |_attempt| { let context = prepare_context.clone(); let policy = prepare_policy.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + let resource = resource_for_prepare.clone(); + async move { + let guard = prepare_segment_origin_attempt(context.clone(), policy, expected_generation).await?; + if !context.session.read().await.transient.origin_resource_is_fetchable(&resource, current_time_millis()) { + finish_segment_origin_attempt(context, guard).await; + return Err(SegmentFetchError::Timeout); + } + Ok(guard) + } + .boxed() }) .await; let response = match response { diff --git a/backend/hls/src/segment_fetcher/origin.rs b/backend/hls/src/segment_fetcher/origin.rs index 0eb8e1754..fa17ab695 100644 --- a/backend/hls/src/segment_fetcher/origin.rs +++ b/backend/hls/src/segment_fetcher/origin.rs @@ -1,16 +1,15 @@ use super::{ - begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, commit_startup_response, - ensure_segment_key_dependency_ready, finish_hls_origin_account_io, hls_object_body_deadline, - reliable_decoded_content_length, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, - HlsBoundAccountAcquireErrorKind, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, - HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, - HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, - SegmentFetchCommit, SegmentFetchContext, SegmentFetchError, SegmentFetchPolicy, SegmentFetchSnapshot, - SegmentOriginWorkFinish, + build_hls_origin_resource_headers, commit_startup_response, ensure_segment_key_dependency_ready, + hls_object_body_deadline, reliable_decoded_content_length, run_hls_origin_resource_retry_loop_with_attempt_prepare, + CachedSegmentMetadata, HlsOriginByteRangeExpectation, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, HlsResourceFetchKind, HlsResourceFetchSource, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, SegmentFetchCommit, SegmentFetchContext, SegmentFetchError, + SegmentFetchPolicy, SegmentFetchSnapshot, SegmentOriginWorkFinish, }; +use crate::origin_attempt::{prepare_media_origin_attempt, HlsMediaOriginAttemptGuard as SegmentOriginAttemptGuard}; use axum::http::HeaderMap; use futures::FutureExt; -use tuliprox_core::utils::{content_coding::DecodedHttpResponse, current_time_millis}; +use tuliprox_core::utils::content_coding::DecodedHttpResponse; use tuliprox_parser::hls::origin_manifest::ParsedByteRange; pub(super) async fn fetch_segment_into_cache( @@ -22,101 +21,26 @@ pub(super) async fn fetch_segment_into_cache( fetch_segment_with_retries_into_cache(context, snapshot, policy).await } -pub(super) struct SegmentOriginAttemptGuard { - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -} - pub(super) async fn prepare_segment_origin_attempt( context: SegmentFetchContext, policy: SegmentFetchPolicy, + expected_generation: u64, ) -> Result { - let started_generation = start_segment_origin_work(&context).await; - let binding = - if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; - let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { - if binding.is_detached() { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); - } - let guard = match begin_hls_origin_account_io_bounded( - origin_io, - &context.session, - binding, - hls_object_body_deadline(policy.origin_segment_timeout_ms), - ) - .await - { - Ok(guard) => guard, - Err(err) => { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(err)); - } - }; - Some((origin_io.clone(), guard)) - } else { - None - }; - Ok(SegmentOriginAttemptGuard { started_generation, provider_lease }) + prepare_media_origin_attempt( + context.origin_io.as_ref(), + &context.session, + expected_generation, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + .map_err(SegmentFetchError::ProviderUnavailable) } pub(super) async fn finish_segment_origin_attempt( context: SegmentFetchContext, guard: SegmentOriginAttemptGuard, ) -> SegmentOriginWorkFinish { - finish_segment_origin_io(&context, guard.started_generation, guard.provider_lease).await -} - -async fn finish_segment_origin_io( - context: &SegmentFetchContext, - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -) -> SegmentOriginWorkFinish { - let origin_work = finish_segment_origin_work(context, started_generation).await; - if let Some((origin_io, guard)) = provider_lease { - finish_hls_origin_account_io( - &origin_io, - &context.session, - guard, - origin_work.generation_valid && origin_work.refresh_reservation, - ) - .await; - touch_segment_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) - .await; - } - origin_work -} - -async fn start_segment_origin_work(context: &SegmentFetchContext) -> Option { - context.origin_io.as_ref()?; - let mut session = context.session.write().await; - Some(session.start_origin_work()) -} - -async fn finish_segment_origin_work( - context: &SegmentFetchContext, - started_generation: Option, -) -> SegmentOriginWorkFinish { - let Some(started_generation) = started_generation else { - return SegmentOriginWorkFinish { generation_valid: true, refresh_reservation: false }; - }; - let mut session = context.session.write().await; - let generation_valid = session.finish_origin_work(started_generation); - let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); - SegmentOriginWorkFinish { generation_valid, refresh_reservation } -} - -async fn touch_segment_origin_account_binding(context: &SegmentFetchContext, reservation_refreshed: bool) { - let mut session = context.session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = current_time_millis(); - binding.last_origin_io_at_ms = Some(now_ms); - if reservation_refreshed { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } + guard.finish(&context.session).await } #[allow(clippy::too_many_lines)] @@ -125,11 +49,8 @@ pub(super) async fn fetch_segment_with_retries_into_cache( snapshot: &SegmentFetchSnapshot, policy: &SegmentFetchPolicy, ) -> Result { - let headers = build_segment_origin_headers( - &context.headers, - &context.origin_provider_session_headers, - snapshot.fetch_ref.byte_range, - )?; + let provider_headers = context.session.read().await.origin_provider_session_headers.clone(); + let headers = build_segment_origin_headers(&context.headers, &provider_headers, snapshot.fetch_ref.byte_range)?; let target = HlsOriginResourceFetchTarget { kind: HlsResourceFetchKind::Segment, source: HlsResourceFetchSource::Normal, @@ -151,6 +72,7 @@ pub(super) async fn fetch_segment_with_retries_into_cache( let session = context.session.read().await; super::super::HlsLogIdentity::from_session(&session) }; + let expected_generation = snapshot.origin_work_generation; let context = context.clone(); let snapshot = snapshot.clone(); let policy_for_prepare = policy.clone(); @@ -164,7 +86,7 @@ pub(super) async fn fetch_segment_with_retries_into_cache( move |_attempt| { let context = prepare_context.clone(); let policy = policy_for_prepare.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + async move { prepare_segment_origin_attempt(context, policy, expected_generation).await }.boxed() }, move |guard| { let context = cleanup_context.clone(); diff --git a/backend/hls/src/segment_fetcher/scheduler.rs b/backend/hls/src/segment_fetcher/scheduler.rs index 99cc90e4f..fbc88bd33 100644 --- a/backend/hls/src/segment_fetcher/scheduler.rs +++ b/backend/hls/src/segment_fetcher/scheduler.rs @@ -107,7 +107,7 @@ pub(super) enum SegmentRetryWake { } #[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum CapacityRetryAdmission { +pub(super) enum CapacityRetryAdmission { Ready, BindingExpired, LocalIoFailure, @@ -189,6 +189,8 @@ impl HlsSegmentWorkerPool { access_leases, metrics, availability_reevaluations, + #[cfg(any(test, feature = "test-support"))] + origin_prepare_barrier: std::sync::Mutex::new(None), } } @@ -202,6 +204,13 @@ impl HlsSegmentWorkerPool { pub fn metrics(&self) -> &Arc { &self.metrics } + #[cfg(any(test, feature = "test-support"))] + pub fn pause_next_origin_prepare_for_test(&self) -> Arc { + let barrier = Arc::new(super::HlsSegmentPrepareBarrier::default()); + *self.origin_prepare_barrier.lock().unwrap() = Some(Arc::clone(&barrier)); + barrier + } + pub async fn classify_backpressure(&self, session: &HlsSessionHandle) -> HlsBackpressureState { let session = session.read().await; self.classify_backpressure_for_session(&session) @@ -217,6 +226,48 @@ impl HlsSegmentWorkerPool { } pub async fn wake_scheduler(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + let (proxy_id, has_work, queued_generation) = { + let session = context.session.read().await; + ( + session.proxy_session_id.clone(), + !session.is_gc_marked_for_removal() && !session.segment_prefetch_queue.is_empty(), + session.activity.origin_work_generation, + ) + }; + #[cfg(any(test, feature = "test-support"))] + let barrier = if has_work { self.origin_prepare_barrier.lock().unwrap().take() } else { None }; + #[cfg(any(test, feature = "test-support"))] + if let Some(barrier) = &barrier { + barrier.entered.notify_one(); + barrier.prepare.notified().await; + } + let has_access = + has_work && self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_id, now_ms); + let preparation_failed = has_access + && crate::origin_account_runtime::prepare_media_origin(context.origin_io.as_ref(), &context.session) + .await + .is_err(); + if has_access && !preparation_failed { + let mut session = context.session.write().await; + // Preparation can invalidate the queue while renewing an account. + // Replan the current qualified window in the new work generation. + // Account handoff clears unqualified fetch refs and keeps this gate + // closed until its replacement manifest has committed. + if session.origin_handoff_ready_after(queued_generation) + && session.origin_account_binding.as_ref().is_none_or(crate::HlsOriginAccountBinding::is_active) + { + let backpressure = self.classify_backpressure_for_session(&session); + session.queue_manifest_fetch_candidates(now_ms, backpressure.allows_prefetch()); + } + } + #[cfg(any(test, feature = "test-support"))] + if let Some(barrier) = &barrier { + barrier.prepared.notify_one(); + barrier.proceed.notified().await; + } + if preparation_failed { + return; + } loop { let runtime = self.runtime.load_full(); let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { @@ -392,7 +443,9 @@ impl HlsSegmentWorkerPool { self.schedule_segment_retry(context, snapshot, retry); return; } - if completion.generation_valid { + if completion.generation_valid + || context.session.read().await.origin_handoff_ready_after(snapshot.origin_work_generation) + { self.schedule_wake(context, finished_at_ms); } } @@ -463,7 +516,7 @@ impl HlsSegmentWorkerPool { } } -async fn wait_for_capacity_retry_admission( +pub(super) async fn wait_for_capacity_retry_admission( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, mut revision: HlsCacheCapacityRevision, @@ -486,7 +539,9 @@ async fn wait_for_capacity_retry_admission( if !binding_current { return CapacityRetryAdmission::BindingExpired; } - match context.segment_cache.ensure_projected_write_capacity(&snapshot.cache_key, projected_write_bytes).await { + let admission = + context.segment_cache.ensure_revision_peak_capacity(&snapshot.cache_key, projected_write_bytes).await; + match admission { Ok(()) => return CapacityRetryAdmission::Ready, Err(error) => { let Some(capacity) = super::super::cache::hls_cache_capacity_from_io(&error) else { diff --git a/backend/hls/src/segment_fetcher/startup_fill.rs b/backend/hls/src/segment_fetcher/startup_fill.rs index 3d48ab8f1..f83c8a0d9 100644 --- a/backend/hls/src/segment_fetcher/startup_fill.rs +++ b/backend/hls/src/segment_fetcher/startup_fill.rs @@ -41,7 +41,7 @@ pub(super) struct HlsObservedFill { pub(super) budget: Arc, pub(super) deadline: Instant, pub(super) repair_permit: Option, - pub(super) additional_reservations: Vec, + pub(super) reservation: Arc, } impl Drop for HlsObservedFill { @@ -56,8 +56,16 @@ impl HlsObservedFill { .checked_add(u64::try_from(chunk.len()).map_err(std::io::Error::other)?) .ok_or_else(|| std::io::Error::other("origin body length overflow"))?; if self.bytes > self.limit { - self.extend_disk_reservation().await?; + self.extend_diskreservation().await?; } + self.context + .segment_cache + .ensure_write_budget_available( + &self.revision.revision().key, + &self.reservation, + self.bytes.saturating_mul(revision_peak_factor(self.file.is_some())), + ) + .await?; if let Some(file) = &mut self.file { file.write_all(chunk).await?; } @@ -104,7 +112,7 @@ impl HlsObservedFill { Ok(()) } - async fn extend_disk_reservation(&mut self) -> std::io::Result<()> { + async fn extend_diskreservation(&mut self) -> std::io::Result<()> { if self.expected.is_some() { return Err(std::io::Error::other("origin body exceeds decoded content length")); } @@ -114,12 +122,10 @@ impl HlsObservedFill { .checked_sub(self.limit) .and_then(|bytes| bytes.checked_mul(factor)) .ok_or_else(|| std::io::Error::other("revision disk reservation overflow"))?; - let mut reservation = - self.context.segment_cache.reserve_revision_peak(&self.revision.revision().key, additional_peak).await?; - if self.file.is_some() { - reservation.mark_unaccounted_writes(); - } - self.additional_reservations.push(reservation); + self.context + .segment_cache + .grow_revision_peak(&self.revision.revision().key, &self.reservation, additional_peak) + .await?; self.limit = next_limit; Ok(()) } @@ -129,10 +135,10 @@ impl HlsObservedFill { if self.bytes == 0 || self.expected.is_some_and(|length| length != self.bytes) { return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete origin segment body")); } - if let Some(file) = self.file.take() { - let mut file = file; + if let Some(file) = &mut self.file { file.flush().await?; - drop(file); + self.file.take(); + self.context.segment_cache.account_reserved_spool(&self.path, self.bytes, &self.reservation)?; self.revision.revision().complete(CachedSegmentMetadata { path: self.path.clone(), size: self.bytes }); self.revision.revision().replay.lock_unpoisoned().take(); } @@ -141,6 +147,7 @@ impl HlsObservedFill { } } +#[allow(clippy::too_many_lines, reason = "Keeps one startup write transaction and its failure closure together.")] pub(super) async fn commit_startup_response( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, @@ -149,6 +156,131 @@ pub(super) async fn commit_startup_response( mut repair_context: HlsSegmentRepairObjectContext, ) -> Result { use futures::StreamExt; + prepare_startup_repair_lease(context, &mut repair_context).await; + let expected = reliable_decoded_content_length(&response); + if response.status != axum::http::StatusCode::OK { + return Err(SegmentFetchError::cache_body(&std::io::Error::other( + "startup requires a complete HTTP 200 response", + ))); + } + let prepared = prepare_startup_fill(context, snapshot, expected, deadline).await?; + let PreparedStartupFill { revision, observed, reservation, raw } = prepared; + let mut transaction = + super::startup_write::StartupWrite::new(context, snapshot, Arc::clone(&reservation), revision.clone()); + transaction.track_fill(&observed); + let result = async { + let chunks_owner = Arc::clone(&observed); + let finish_owner = Arc::clone(&observed); + let stream = tokio_util::io::ReaderStream::new(response.body) + .then(move |chunk| { + let observed = Arc::clone(&chunks_owner); + async move { + let chunk = chunk?; + observed.lock().await.observe(&chunk).await?; + Ok::<_, std::io::Error>(chunk) + } + }) + .chain(futures::stream::once(async move { + finish_owner.lock().await.finish().await?; + Ok::<_, std::io::Error>(bytes::Bytes::new()) + })); + let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::BeforeNormalCommit) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + let mut metadata = context + .segment_repair + .commit_origin_response_with_budget( + &context.segment_cache, + &snapshot.cache_key, + reader, + deadline, + repair_context.clone(), + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::AfterNormalCommit) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + if !raw { + context + .segment_repair + .repair_ready_cache_hit_with_budget( + &context.segment_cache, + &snapshot.cache_key, + repair_context, + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + let latest = context + .segment_cache + .metadata(&snapshot.cache_key) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))? + .ok_or_else(|| { + SegmentFetchError::cache_body(&std::io::Error::other("processed cache object disappeared")) + })?; + metadata = latest; + let processed = context + .segment_cache + .publish_processed_revision_with_budget( + &revision.revision().key, + &metadata.path, + deadline, + Some(Arc::clone(&reservation)), + ) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + revision.revision().complete(processed); + } + #[cfg(test)] + context + .segment_cache + .startup_write_checkpoint(crate::cache::StartupWritePhase::AfterPublication) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + Ok(metadata) + } + .await; + if result.is_ok() { + transaction.finish(); + } else { + transaction.abort().await; + } + // A new origin attempt needs its entire peak again, even when the failed + // operation was a smaller revision file or a budget extension. + let peak = { + let fill = observed.lock().await; + let retry_bound = + if fill.bytes > fill.limit { fill.limit.saturating_mul(2).max(fill.bytes) } else { fill.limit }; + retry_bound.saturating_mul(revision_peak_factor(raw)) + }; + drop(observed); + result.map_err(|error| match error { + SegmentFetchError::LocalCacheCapacity { required_session_bytes, required_global_bytes, revision, .. } => { + SegmentFetchError::LocalCacheCapacity { + required_session_bytes, + required_global_bytes, + projected_write_bytes: peak, + revision, + } + } + other => other, + }) +} + +async fn prepare_startup_repair_lease( + context: &SegmentFetchContext, + repair_context: &mut HlsSegmentRepairObjectContext, +) { if repair_context.hls_access_lease_id.is_none() { let leases = context.session.read().await.startup.as_ref().map(|startup| Arc::clone(&startup.access_leases)); if let Some(leases) = leases { @@ -161,65 +293,12 @@ pub(super) async fn commit_startup_response( if let Some(lease_id) = &repair_context.hls_access_lease_id { context.segment_repair.ensure_access_lease_window(lease_id.clone()).await; } - let expected = reliable_decoded_content_length(&response); - if response.status != axum::http::StatusCode::OK { - return Err(SegmentFetchError::cache_body(&std::io::Error::other( - "startup requires a complete HTTP 200 response", - ))); - } - let prepared = prepare_startup_fill(context, snapshot, expected, deadline).await?; - let PreparedStartupFill { revision, observed, _reservation, raw } = prepared; - let chunks_owner = Arc::clone(&observed); - let finish_owner = Arc::clone(&observed); - let stream = tokio_util::io::ReaderStream::new(response.body) - .then(move |chunk| { - let observed = Arc::clone(&chunks_owner); - async move { - let chunk = chunk?; - observed.lock().await.observe(&chunk).await?; - Ok::<_, std::io::Error>(chunk) - } - }) - .chain(futures::stream::once(async move { - finish_owner.lock().await.finish().await?; - Ok::<_, std::io::Error>(bytes::Bytes::new()) - })); - let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); - let mut metadata = context - .segment_repair - .commit_origin_response(&context.segment_cache, &snapshot.cache_key, reader, deadline, repair_context.clone()) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - if !raw { - context - .segment_repair - .repair_ready_cache_hit(&context.segment_cache, &snapshot.cache_key, repair_context) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - let latest = context - .segment_cache - .metadata(&snapshot.cache_key) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))? - .ok_or_else(|| { - SegmentFetchError::cache_body(&std::io::Error::other("processed cache object disappeared")) - })?; - metadata = latest; - let processed = context - .segment_cache - .publish_processed_revision(&revision.revision().key, &metadata.path, deadline) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - revision.revision().complete(processed); - } - drop(observed); - Ok(metadata) } pub(super) struct PreparedStartupFill { pub(super) revision: super::super::SegmentRevisionGuard, pub(super) observed: Arc>, - pub(super) _reservation: super::super::HlsRevisionDiskReservation, + pub(super) reservation: Arc, pub(super) raw: bool, } @@ -238,6 +317,7 @@ pub(super) async fn prepare_startup_fill( } } +#[allow(clippy::too_many_lines)] async fn try_prepare_startup_fill( context: &SegmentFetchContext, snapshot: &SegmentFetchSnapshot, @@ -283,7 +363,8 @@ async fn try_prepare_startup_fill( let peak = body_bound .checked_mul(revision_peak_factor(raw)) .ok_or_else(|| SegmentFetchError::cache_body(&std::io::Error::other("revision peak overflow")))?; - let mut reservation = match context.segment_cache.reserve_revision_peak(&revision.revision().key, peak).await { + let mut reservation = match context.segment_cache.reserve_revision_peak_reclaiming(&snapshot.cache_key, peak).await + { Ok(reservation) => reservation, Err(_) if raw && decides_policy && fallback_to_first_ready(&context.session).await => { debug!("HLS startup fallback: reason=disk_capacity effective_mode=first_ready"); @@ -291,24 +372,44 @@ async fn try_prepare_startup_fill( } Err(error) => return Err(SegmentFetchError::cache_body(&error)), }; - let path = context.segment_cache.object_path(&revision.revision().key); + let path = reservation.object_path(&revision.revision().key); + if raw { + reservation.mark_unaccounted_writes(); + } + let reservation = Arc::new(reservation); + let mut preparation = + super::startup_write::StartupWrite::new(context, snapshot, Arc::clone(&reservation), revision.clone()); let file = if raw { - let (file, pin) = context + let (file, pin) = match context .segment_cache - .create_revision_spool(&revision.revision().key) + .create_revision_spool_with_budget(&revision.revision().key, Some(&reservation)) .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; + { + Ok(spool) => spool, + Err(error) => { + preparation.abort().await; + return Err(SegmentFetchError::cache_body(&error)); + } + }; *revision.revision().file_pin.lock_unpoisoned() = Some(pin); *revision.revision().replay.lock_unpoisoned() = replay.take(); - reservation.mark_unaccounted_writes(); Some(file) } else { - let pin = - context.segment_cache.pin_revision_file(&path).map_err(|error| SegmentFetchError::cache_body(&error))?; + let pin = match context.segment_cache.pin_revision_file(&path) { + Ok(pin) => pin, + Err(error) => { + preparation.abort().await; + return Err(SegmentFetchError::cache_body(&error)); + } + }; *revision.revision().file_pin.lock_unpoisoned() = Some(pin); None }; - bind_startup_revision(context, snapshot, &revision).await?; + if let Err(error) = bind_startup_revision(context, snapshot, &revision).await { + preparation.abort().await; + return Err(error); + } + preparation.finish(); debug!("HLS startup fill: mode={mode} proxy_seq={} representation={kind:?}", snapshot.proxy_seq); let observed = Arc::new(tokio::sync::Mutex::new(HlsObservedFill { revision: revision.clone(), @@ -323,9 +424,9 @@ async fn try_prepare_startup_fill( budget, deadline, repair_permit: None, - additional_reservations: Vec::new(), + reservation: Arc::clone(&reservation), })); - Ok(Some(PreparedStartupFill { revision, observed, _reservation: reservation, raw })) + Ok(Some(PreparedStartupFill { revision, observed, reservation, raw })) } async fn bind_startup_revision( diff --git a/backend/hls/src/segment_fetcher/startup_write.rs b/backend/hls/src/segment_fetcher/startup_write.rs new file mode 100644 index 000000000..8f9debf9e --- /dev/null +++ b/backend/hls/src/segment_fetcher/startup_write.rs @@ -0,0 +1,105 @@ +//! Failure/cancellation closure for one startup write, keyed by its peak owner. +use super::{segment_fetch_attempt_matches, SegmentFetchContext, SegmentFetchSnapshot}; +use crate::{sync_ext::MutexExt, HlsRevisionDiskReservation, SegmentRevisionGuard, SegmentRevisionState}; +use std::sync::Arc; + +/// Rolls back only this attempt and keeps immutable completed revisions alive. +pub(super) struct StartupWrite { + context: SegmentFetchContext, + snapshot: SegmentFetchSnapshot, + budget: Arc, + revision: SegmentRevisionGuard, + armed: bool, + observed: Option>>, +} + +impl StartupWrite { + pub(super) fn new( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + budget: Arc, + revision: SegmentRevisionGuard, + ) -> Self { + Self { context: context.clone(), snapshot: snapshot.clone(), budget, revision, armed: true, observed: None } + } + + pub(super) fn track_fill(&mut self, fill: &Arc>) { + self.observed = Some(Arc::clone(fill)); + } + + pub(super) fn finish(&mut self) { self.armed = false; } + + pub(super) async fn abort(&mut self) { + cleanup(&self.context, &self.snapshot, &self.budget, &self.revision, self.observed.as_ref()).await; + self.armed = false; + } +} + +impl Drop for StartupWrite { + fn drop(&mut self) { + if !self.armed { + return; + } + let context = self.context.clone(); + let snapshot = self.snapshot.clone(); + let budget = Arc::clone(&self.budget); + let revision = self.revision.clone(); + let observed = self.observed.clone(); + revision.revision().fail(); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(async move { + cleanup(&context, &snapshot, &budget, &revision, observed.as_ref()).await; + let mut session = context.session.write().await; + session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); + if session + .segments + .get(&snapshot.proxy_seq) + .is_some_and(|entry| segment_fetch_attempt_matches(entry, &snapshot)) + { + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = crate::SegmentCacheStatus::Discovered; + } + if let Some(notifier) = session.segment_fetch_notifiers.remove(&snapshot.proxy_seq) { + notifier.notify_waiters(); + } + } + }); + } + } +} + +async fn cleanup( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + budget: &HlsRevisionDiskReservation, + revision: &SegmentRevisionGuard, + observed: Option<&Arc>>, +) { + if let Some(observed) = observed { + use tokio::io::AsyncWriteExt; + let mut fill = observed.lock().await; + if let Some(mut file) = fill.file.take() { + let _ = file.flush().await; + } + } + revision.revision().fail(); + if let Err(error) = context.segment_cache.delete_write_owned(&snapshot.cache_key, budget).await { + log::warn!( + "HLS startup partial commit cleanup deferred: proxy_seq={} error_kind={:?}", + snapshot.proxy_seq, + error.kind() + ); + } + // Complete revisions have immutable, independently pinned bytes. Failed + // spools are unique to this revision and cannot be confused with a retry. + if matches!(revision.revision().state(), SegmentRevisionState::Failed) { + revision.revision().file_pin.lock_unpoisoned().take(); + if let Err(error) = context.segment_cache.delete_write_owned(&revision.revision().key, budget).await { + log::warn!( + "HLS startup failed revision cleanup deferred: proxy_seq={} error_kind={:?}", + snapshot.proxy_seq, + error.kind() + ); + } + } +} diff --git a/backend/hls/src/segment_fetcher/tests.rs b/backend/hls/src/segment_fetcher/tests.rs index e3fc02b48..dea31ae05 100644 --- a/backend/hls/src/segment_fetcher/tests.rs +++ b/backend/hls/src/segment_fetcher/tests.rs @@ -12,6 +12,7 @@ mod protocol; mod query; mod retry; mod startup; +mod startup_capacity; mod support; use self::support::{ diff --git a/backend/hls/src/segment_fetcher/tests/behavior.rs b/backend/hls/src/segment_fetcher/tests/behavior.rs index f77768134..68caa5586 100644 --- a/backend/hls/src/segment_fetcher/tests/behavior.rs +++ b/backend/hls/src/segment_fetcher/tests/behavior.rs @@ -428,7 +428,7 @@ async fn same_origin_sequence_resource_conflict_preserves_in_flight_binding() { let (committed_key, bytes) = committed_segment(&context, 1).await; assert_eq!(committed_key, cache_key); assert_eq!(bytes, b"controlled-segment"); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] @@ -495,7 +495,7 @@ async fn segment_decoder_failure_exhausts_attempt_budget_without_cache_or_temp_f let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] @@ -662,7 +662,7 @@ async fn unknown_body_reserves_more_disk_before_growing_its_raw_spool() -> std:: let mut observed = prepared.observed.lock().await; observed.limit = 8; observed.observe(b"segment-body").await?; - assert_eq!(observed.additional_reservations.len(), 1); + assert_eq!(observed.limit, 16); observed.finish().await?; let metadata = prepared.revision.revision().wait_complete(tokio::time::Instant::now() + Duration::from_secs(1)).await?; diff --git a/backend/hls/src/segment_fetcher/tests/lifecycle.rs b/backend/hls/src/segment_fetcher/tests/lifecycle.rs index 6f3ddbe60..018b0e006 100644 --- a/backend/hls/src/segment_fetcher/tests/lifecycle.rs +++ b/backend/hls/src/segment_fetcher/tests/lifecycle.rs @@ -2,10 +2,253 @@ use super::{ encrypted_fetch_context, fetch_context, spawn_segment_server, spawn_sequence_response_server, take_queued_segment_fetch_candidate, SegmentFetchPolicy, SegmentFetchPriority, TestOriginResponse, }; -use crate::{HlsSegmentFile, SegmentCacheStatus, TransientResourceKind}; +use crate::{ + api::{ + HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount, + LiveHlsOriginEntry, + }, + HlsSegmentFile, SegmentCacheStatus, TransientResourceKind, +}; use std::time::Duration; use tuliprox_core::utils::current_time_millis; +#[derive(Debug)] +struct InvalidatingPrepare { + first: std::sync::atomic::AtomicBool, + handoff: bool, +} + +impl HlsOriginAccountRuntime for InvalidatingPrepare { + fn prepare<'a>( + &'a self, + session: &'a crate::HlsSessionHandle, + _work: HlsOriginAccountWork, + _class: crate::HlsOriginWorkClass, + ) -> std::pin::Pin< + Box< + dyn std::future::Future> + + Send + + 'a, + >, + > { + Box::pin(async move { + let mut s = session.write().await; + if self.first.swap(false, std::sync::atomic::Ordering::AcqRel) { + if self.handoff { + let binding = crate::HlsOriginAccountBinding::new( + "input".into(), + "new-account".into(), + &s.proxy_session_id, + current_time_millis(), + ); + s.handoff_origin_account(binding); + } else { + s.invalidate_queued_origin_work(); + } + } + Ok(HlsPreparedOriginAccount { + entry: LiveHlsOriginEntry::parse("http://unused.example/index.m3u8").unwrap(), + snapshot: HlsOriginAccountSnapshot::from_session(&s), + }) + }) + } +} + +#[tokio::test] +async fn scheduler_replans_valid_manifest_work_after_prepare_invalidates_generation() { + use std::sync::{atomic::AtomicBool, Arc}; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + for handoff in [false, true] { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy::default(); + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + super::support::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + current_time_millis(), + ) + .await; + let original = context.session.read().await.segment_prefetch_queue.proxy_seqs(); + assert_eq!(original.len(), 3); + let runtime = Arc::new(InvalidatingPrepare { first: AtomicBool::new(true), handoff }); + context.origin_io = Some(crate::HlsOriginIoContext { + ctx: crate::HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }); + worker.wake_scheduler(context.clone(), current_time_millis()).await; + assert!(!runtime.first.load(std::sync::atomic::Ordering::Acquire), "account preparation actually ran"); + assert_eq!(context.session.read().await.activity.origin_work_generation, 1); + if handoff { + let s = context.session.read().await; + assert!(s.origin_account_manifest_pending.is_some()); + assert!(s.segment_prefetch_queue.is_empty()); + assert!(original.iter().all(|seq| s.segments[seq].origin_fetch_ref.is_none())); + assert!(server.requests.lock().await.is_empty(), "unqualified old-account resources stay invalid"); + } else { + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let s = context.session.read().await; + if original.iter().all(|seq| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) { + break; + } + drop(s); + tokio::task::yield_now().await; + } + }) + .await + .expect("valid jobs survive invalidation in prepare without test-side requeue"); + assert_eq!(server.requests.lock().await.len(), original.len()); + } + } +} + +#[tokio::test] +async fn scheduler_replan_uses_current_backpressure_for_prefetch() { + use crate::HlsBackpressureState; + use std::sync::{atomic::AtomicBool, Arc}; + use tuliprox_core::model::{Config, PlaybackKind}; + use tuliprox_session::ConnectionKind; + + for expected_pressure in + [HlsBackpressureState::Degraded, HlsBackpressureState::Saturated, HlsBackpressureState::Normal] + { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy { max_global_segment_fetches: 2, ..SegmentFetchPolicy::default() }; + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + super::support::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + current_time_millis(), + ) + .await; + let semaphore = Arc::clone(&worker.runtime.load().global_semaphore); + let mut held_permits = vec![semaphore.try_acquire().unwrap()]; + let previous_generation = { + let mut s = context.session.write().await; + s.invalidate_queued_origin_work(); + s.initial_prefetch_gap_segments = 1; + let pressure = worker.classify_backpressure_for_session(&s); + assert_eq!(pressure, HlsBackpressureState::Degraded); + let report = s.queue_manifest_fetch_candidates(current_time_millis(), pressure.allows_prefetch()); + assert_eq!(report.render_window_queued, 2); + assert_eq!(report.prefetch_queued, 0); + assert_eq!(report.prefetch_skipped, 1); + s.activity.origin_work_generation + }; + let runtime = Arc::new(InvalidatingPrepare { first: AtomicBool::new(true), handoff: false }); + context.origin_io = Some(crate::HlsOriginIoContext { + ctx: crate::HlsCtx::for_test(Config::default()), + client_addr: "127.0.0.1:12345".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 1, + preacquired_provider_handle: None, + started_generation: None, + prepared_account: None, + playback_kind: PlaybackKind::LiveHls, + account_runtime: Some(runtime.clone()), + }); + let barrier = worker.pause_next_origin_prepare_for_test(); + let scheduler_worker = Arc::clone(&worker); + let scheduler_context = context.clone(); + let task = tokio::spawn(async move { + scheduler_worker.wake_scheduler(scheduler_context, current_time_millis()).await; + }); + tokio::time::timeout(Duration::from_secs(3), barrier.entered.notified()).await.unwrap(); + // Pressure can change after the manifest commit. Replanning must use its + // current classification, including recovery back to Normal. + match expected_pressure { + HlsBackpressureState::Normal => held_permits.clear(), + HlsBackpressureState::Degraded => {} + HlsBackpressureState::Saturated => held_permits.push(semaphore.try_acquire().unwrap()), + } + barrier.prepare.notify_one(); + tokio::time::timeout(Duration::from_secs(3), barrier.prepared.notified()).await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled(), "observe the replan before workers consume its queue"); + + assert!(!runtime.first.load(std::sync::atomic::Ordering::Acquire), "account preparation actually ran"); + let s = context.session.read().await; + assert_eq!(s.activity.origin_work_generation, previous_generation + 1); + let pressure = worker.classify_backpressure_for_session(&s); + assert_eq!(pressure, expected_pressure); + for seq in [1, 2] { + assert!(s.segment_prefetch_queue.contains(seq), "{pressure:?}: restore render-window work"); + assert!(matches!( + s.segments[&seq].status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. } + )); + } + let prefetch_queued = s.segment_prefetch_queue.contains(3); + eprintln!( + "replan pressure={pressure:?} generation={} render_window=[1, 2] prefetch_queued={prefetch_queued}", + s.activity.origin_work_generation + ); + assert_eq!( + prefetch_queued, + pressure.allows_prefetch(), + "{pressure:?}: replan must respect the current prefetch policy" + ); + assert_eq!( + matches!( + s.segments[&3].status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, .. } + ), + pressure.allows_prefetch() + ); + assert_eq!(s.segment_prefetch_queue.len(), if pressure.allows_prefetch() { 3 } else { 2 }); + assert!(server.requests.lock().await.is_empty(), "the barrier precedes origin segment work"); + } +} + +#[tokio::test] +async fn account_handoff_during_segment_prepare_blocks_old_origin_request() { + use crate::{origin_attempt::tests::bound_origin_io, HlsOriginAccountIoLease, HlsOriginResourceFetchError}; + use std::sync::{atomic::Ordering, Arc}; + + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, mut context, _) = fetch_context(&server, &temp_dir, &policy).await; + let (io, replacement) = bound_origin_io(&context.session).await; + context.origin_io = Some(io); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.expect("segment snapshot"); + let replacement_lease = Arc::new(HlsOriginAccountIoLease::active_for_test(&replacement, 1)); + let read_blocker = context.session.read().await; + let mut fetch = Box::pin(super::super::fetch_segment_with_retries_into_cache(&context, &snapshot, &policy)); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + let mut handoff = Box::pin(context.session.write()); + assert!(futures::poll!(handoff.as_mut()).is_pending()); + drop(read_blocker); + assert!(futures::poll!(fetch.as_mut()).is_pending()); + { + let mut session = handoff.await; + session.handoff_origin_account(replacement); + session.origin_account_io_lease = Some(Arc::clone(&replacement_lease)); + } + + let result = fetch.await; + assert!(server.requests.lock().await.is_empty(), "stale segment URL must never reach the origin"); + assert!(matches!(result, Err(HlsOriginResourceFetchError::ProviderUnavailable(_)))); + let session = context.session.read().await; + assert_eq!(session.activity.active_origin_work_count.load(Ordering::Acquire), 0); + assert_eq!(replacement_lease.active_io_count.load(Ordering::Acquire), 1); +} + #[tokio::test] async fn popped_candidate_with_expired_fetch_binding_returns_to_discovered() { let server = spawn_sequence_response_server(vec![TestOriginResponse { diff --git a/backend/hls/src/segment_fetcher/tests/persistence.rs b/backend/hls/src/segment_fetcher/tests/persistence.rs index 1da9dae2a..ebf3debc3 100644 --- a/backend/hls/src/segment_fetcher/tests/persistence.rs +++ b/backend/hls/src/segment_fetcher/tests/persistence.rs @@ -88,5 +88,5 @@ async fn segment_decoder_failure_retries_then_commits_and_cleans_temp_file() { assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); drop(requests); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } diff --git a/backend/hls/src/segment_fetcher/tests/retry.rs b/backend/hls/src/segment_fetcher/tests/retry.rs index 118135991..9b345aebd 100644 --- a/backend/hls/src/segment_fetcher/tests/retry.rs +++ b/backend/hls/src/segment_fetcher/tests/retry.rs @@ -399,7 +399,7 @@ async fn segment_decoded_object_limit_is_authoritative_and_non_retryable() { let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); + assert_eq!(temp_cache_files(temp_dir.path()).len(), 0); } #[tokio::test] diff --git a/backend/hls/src/segment_fetcher/tests/startup_capacity.rs b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs new file mode 100644 index 000000000..9ca91015b --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs @@ -0,0 +1,394 @@ +use super::*; +use crate::{cache::StartupWritePhase, SegmentCacheStatus}; +use std::{fmt::Write as _, sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; + +#[tokio::test] +async fn startup_failure_and_cancellation_close_each_commit_transition() -> std::io::Result<()> { + for phase in [ + StartupWritePhase::BeforeNormalCommit, + StartupWritePhase::AfterNormalCommit, + StartupWritePhase::AfterPublication, + ] { + for cancel in [false, true] { + startup_transition_round(phase, cancel).await?; + } + } + Ok(()) +} + +async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std::io::Result<()> { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + let (entered, resume) = context.segment_cache.pause_startup_write(phase); + let task_context = context.clone(); + let task_snapshot = snapshot.clone(); + let task_policy = policy.clone(); + let task = tokio::spawn(async move { + super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await + }); + tokio::time::timeout(Duration::from_secs(5), entered).await.unwrap().unwrap(); + let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&1].clone(); + if cancel { + task.abort(); + assert!(matches!(task.await, Err(error) if error.is_cancelled())); + drop(resume); + } else { + drop(resume); // A closed checkpoint injects a local I/O failure. + let result = task.await.unwrap(); + assert!(result.is_err()); + let mut session = context.session.write().await; + worker.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, 11); + } + let id = snapshot.cache_key.proxy_session_id(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if context.segment_cache.remaining_revision_bytes(id) == 0 + && context.session.read().await.active_segment_fetches == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("bounded cleanup completes without GC or foreign mutations"); + assert!(context.segment_cache.metadata(&snapshot.cache_key).await?.is_none()); + assert_eq!(revision.revision().is_complete(), phase == StartupWritePhase::AfterPublication); + let bytes = context.segment_cache.capacity_usage(id).await?.session_bytes; + assert_eq!(bytes, if phase == StartupWritePhase::AfterPublication { 10 } else { 0 }); + assert_eq!(context.segment_cache.revision_disk_bytes(), context.segment_cache.scan_revision_disk_bytes().await?); + assert_eq!(temp_cache_files(directory.path()).len(), 0); + if phase == StartupWritePhase::AfterPublication { + for _ in 0..2 { + let metadata = revision.revision().wait_complete(tokio::time::Instant::now()).await?; + assert_eq!(tokio::fs::read(metadata.path).await?, b"body:/1.ts"); + } + } + Ok(()) +} + +#[tokio::test] +async fn capacity_retry_requires_the_complete_peak_and_only_reservation_release() -> std::io::Result<()> { + use super::super::scheduler::{wait_for_capacity_retry_admission, CapacityRetryAdmission}; + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + context.segment_cache.update_cache_limits(100, 100); + let held = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 70).await?; + let small = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 10).await?; + let sufficient = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 10).await?; + let error = context.segment_cache.reserve_revision_peak(&snapshot.cache_key, 30).await.err().unwrap(); + let revision = crate::cache::hls_cache_capacity_from_io(&error).unwrap().revision().clone(); + context.session.write().await.segments.get_mut(&1).unwrap().status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 11 }; + let waiting = wait_for_capacity_retry_admission(&context, &snapshot, revision, 30); + tokio::pin!(waiting); + assert!(futures::poll!(waiting.as_mut()).is_pending()); + drop(small); + assert!(futures::poll!(waiting.as_mut()).is_pending(), "20 free bytes cannot admit a fresh 30-byte peak"); + assert!(server.requests.lock().await.is_empty()); + drop(sufficient); + assert_eq!(tokio::time::timeout(Duration::from_secs(2), waiting).await.unwrap(), CapacityRetryAdmission::Ready); + assert!(server.requests.lock().await.is_empty()); + drop(held); + Ok(()) +} + +#[tokio::test] +async fn thousand_live_windows_keep_one_manager_two_leases_and_bounded_revision_disk() -> std::io::Result<()> { + tokio::time::timeout(Duration::from_secs(60), live_window_soak()) + .await + .expect("1000-window soak makes bounded progress") +} + +#[allow(clippy::too_many_lines)] +async fn live_window_soak() -> std::io::Result<()> { + use crate::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, + }; + use shared::model::{ByteSize, HlsCacheConfigDto, HlsStartupMode}; + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + cache_duration: shared::model::Secs::new(1), + cache_bytes: ByteSize::new("512 B"), + cache_bytes_per_session: ByteSize::new("256 B"), + max_concurrent_segment_fetches_per_session: 1, + ..Default::default() + }; + dto.startup.mode = HlsStartupMode::FirstReady; + let manager = HlsProxyManager::with_hls_cache_config(&tuliprox_core::model::HlsCacheConfig::from(&dto)); + let now = current_time_millis(); + let session_key = HlsSessionKey::new(1, "capacity-soak"); + let (session, _) = manager.get_or_create_session_with_outcome(session_key.clone(), b"secret", now).await; + let id = session.read().await.proxy_session_id.clone(); + let lease_ids = [HlsAccessLeaseId("soak-a".into()), HlsAccessLeaseId("soak-b".into())]; + for lease_id in &lease_ids { + let mut lease = HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("test", &lease_id.0), + id.clone(), + "test".into(), + "test-session".into(), + 1, + "capacity-soak".into(), + 1, + now, + 1_000_000, + ); + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now + 1_000_000); + assert!(manager.access_leases().write().await.prepare_access_lease(lease)); + } + let (_, mut context, _) = fetch_context(&server, &directory, &SegmentFetchPolicy::default()).await; + context.session = Arc::clone(&session); + context.segment_cache = Arc::clone(manager.segment_cache()); + context.segment_repair = Arc::clone(manager.segment_repair()); + let worker = Arc::clone(manager.segment_worker_pool()); + let policy = worker.policy(); + let mut downloads = 0usize; + let mut injected_failures = 0usize; + let mut reclaimed = 0u64; + for head in 0u64..1000 { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n"); + for seq in head..head + 6 { + writeln!(body, "#EXTINF:4,\n{}/{seq}.ts", server.base_url).unwrap(); + } + let keys = { + let mut guard = session.write().await; + guard.last_client_access_at_ms = now + head; + guard.apply_origin_manifest(&normal_manifest(&body)).unwrap(); + guard + .segments + .values() + .filter(|entry| { + entry.origin_fetch_ref.is_some() && !matches!(entry.status, SegmentCacheStatus::Ready { .. }) + }) + .map(|entry| entry.cache_key.clone()) + .collect::>() + }; + for key in keys { + let inject = head > 0 && head.is_multiple_of(100); + if inject { + let (entered, resume) = context.segment_cache.pause_startup_write(StartupWritePhase::AfterNormalCommit); + let fill = soak_fetch(&worker, &context, &policy, key.proxy_seq(), now + head); + tokio::pin!(fill); + tokio::select! { + reached = entered => reached.unwrap(), + result = &mut fill => panic!("fill bypassed injected checkpoint: {}", result.is_ok()), + } + drop(resume); + assert!(fill.await.is_err()); + injected_failures += 1; + session.write().await.segments.get_mut(&key.proxy_seq()).unwrap().status = + SegmentCacheStatus::Discovered; + } + soak_fetch(&worker, &context, &policy, key.proxy_seq(), now + head).await?; + downloads += 1; + let segment = crate::HlsSegmentFile { proxy_seq: key.proxy_seq(), extension: "ts".into() }; + for _ in 0..2 { + assert_eq!( + worker.demand_fetch_and_wait(context.clone(), &segment, now + head).await, + super::super::SegmentDemandFetchOutcome::Ready + ); + } + } + let snapshot = { + let mut guard = session.write().await; + guard.render_and_store_manifest(now + head).unwrap(); + let rendered = guard.last_rendered_manifest.as_ref().unwrap(); + crate::manifest_snapshot::derive_hls_lease_manifest_snapshot( + &crate::manifest_snapshot::HlsLeaseManifestSnapshotInput::NormalCacheTimeline { + session: &guard, + committed_body: &rendered.body, + materialized_body: &rendered.body, + stripped_tail_segments: 0, + }, + now + head, + ) + .unwrap() + .unwrap() + }; + for lease_id in &lease_ids { + let expected = manager.prepare_access_lease_manifest_publication(lease_id, &id, now + head).await.unwrap(); + assert!(manager + .commit_access_lease_manifest_publication(lease_id, &id, expected, snapshot.clone(), now + head) + .await + .is_committed()); + } + drop(snapshot); + let report = manager.run_garbage_collection_once(now + head + 60_000).await?; + reclaimed += report.revision_bytes_reclaimed; + let usage = context.segment_cache.capacity_usage(&id).await?; + assert!(usage.session_bytes <= 256 && usage.global_bytes <= 512); + assert_eq!(context.segment_cache.remaining_revision_bytes(&id), 0); + assert_eq!( + context.segment_cache.revision_disk_bytes(), + context.segment_cache.scan_revision_disk_bytes().await? + ); + if head.is_multiple_of(100) { + eprintln!("live-window={head} downloads={downloads} revision_gc_bytes={reclaimed} injected_failures={injected_failures}"); + } + } + for lease in &lease_ids { + manager.access_leases().write().await.remove_access_lease(lease); + } + for _ in 0..16 { + if manager.run_garbage_collection_once(now + 61_000).await?.revision_deletions_planned == 0 { + break; + } + } + let guard = session.read().await; + let startup = guard.startup.as_ref().unwrap(); + assert!(startup.store.retire_unpinned()?.is_empty()); + let retained = startup + .revisions + .values() + .filter_map(|owner| match owner.revision().state() { + crate::SegmentRevisionState::Complete(metadata) => Some(metadata.size), + crate::SegmentRevisionState::Pending | crate::SegmentRevisionState::Failed => None, + }) + .sum::(); + assert_eq!(context.segment_cache.scan_revision_disk_bytes().await?, retained); + assert_eq!(context.segment_cache.remaining_revision_bytes(&id), 0); + assert!(downloads >= 1000 && injected_failures >= 9 && reclaimed > 0); + drop(guard); + let (same_session, _) = manager.get_or_create_session_with_outcome(session_key, b"secret", now + 1001).await; + assert!(Arc::ptr_eq(&session, &same_session)); + assert_eq!(server.requests.lock().await.len(), downloads + injected_failures); + eprintln!( + "live-window=1000 downloads={downloads} revision_gc_bytes={reclaimed} injected_failures={injected_failures}" + ); + Ok(()) +} + +async fn soak_fetch( + worker: &Arc, + context: &SegmentFetchContext, + policy: &SegmentFetchPolicy, + seq: u64, + now: u64, +) -> std::io::Result<()> { + assert!(context.session.write().await.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Demand, now)); + let snapshot = worker.next_fetch_snapshot(context, now, policy).await.unwrap(); + let result = super::super::fetch_segment_into_cache(context, &snapshot, policy).await; + let success = result.is_ok(); + let mut session = context.session.write().await; + worker.apply_segment_fetch_result(&mut session, &snapshot, policy, result, now); + if success { + Ok(()) + } else { + Err(std::io::Error::other("injected or unexpected fill failure")) + } +} + +#[tokio::test] +async fn processed_peak_release_resumes_a_real_deferred_download_without_gc() -> std::io::Result<()> { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, segment) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; + clear_scheduled_prefetch(&context, &policy).await; + context.segment_cache.update_cache_limits(100, 100); + let key = context.session.read().await.segments[&1].cache_key.clone(); + let held = context.segment_cache.reserve_revision_peak(&key, 80).await?; + let demand = worker.demand_fetch_and_wait(context.clone(), &segment, current_time_millis()); + tokio::pin!(demand); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + tokio::select! { + outcome = &mut demand => panic!("premature demand completion: {outcome:?}"), + () = tokio::task::yield_now() => {}, + } + if matches!(context.session.read().await.segments[&1].status, SegmentCacheStatus::CapacityDeferred { .. }) { + break; + } + } + }) + .await + .unwrap(); + assert_eq!(server.requests.lock().await.len(), 1); + drop(held); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), demand).await.unwrap(), + super::super::SegmentDemandFetchOutcome::Ready + ); + assert_eq!(server.requests.lock().await.len(), 2); + assert_eq!(context.segment_cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert_eq!(tokio::fs::read(context.segment_cache.object_path(&key)).await?, b"body:/1.ts"); + Ok(()) +} + +#[tokio::test] +async fn unknown_and_growing_bodies_abort_before_writing_unreserved_bytes() -> std::io::Result<()> { + use futures::StreamExt; + for mode in [shared::model::HlsStartupMode::FirstReady, shared::model::HlsStartupMode::Progressive] { + for expected in [None, Some(8)] { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, mode).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.startup.as_mut().unwrap().config.max_progressive_bytes_per_segment = + shared::model::Bytes::new(8); + context.segment_cache.update_cache_limits(45, 45); + context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let prepared = super::super::prepare_startup_fill(&context, &snapshot, expected, deadline) + .await + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let mut transaction = super::super::startup_write::StartupWrite::new( + &context, + &snapshot, + Arc::clone(&prepared.reservation), + prepared.revision.clone(), + ); + transaction.track_fill(&prepared.observed); + let observed = Arc::clone(&prepared.observed); + let stream = futures::stream::iter([b"12345678".as_slice(), b"extra".as_slice()]).then(move |chunk| { + let observed = Arc::clone(&observed); + async move { + observed.lock().await.observe(chunk).await?; + Ok::<_, std::io::Error>(bytes::Bytes::copy_from_slice(chunk)) + } + }); + let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); + let result = context + .segment_cache + .stage_temp_with_budget(&snapshot.cache_key, reader, deadline, Some(Arc::clone(&prepared.reservation))) + .await; + assert!(result.is_err()); + // The first chunk was covered; the rejected second chunk never + // reached either the normal staging file or the raw spool. + if prepared.raw { + assert_eq!(tokio::fs::metadata(&prepared.observed.lock().await.path).await?.len(), 8); + } + transaction.abort().await; + drop(transaction); + drop(prepared); + assert_eq!( + context.segment_cache.capacity_usage(snapshot.cache_key.proxy_session_id()).await?.session_bytes, + 0 + ); + assert_eq!(context.segment_cache.remaining_revision_bytes(snapshot.cache_key.proxy_session_id()), 0); + assert_eq!(temp_cache_files(directory.path()).len(), 0); + } + } + Ok(()) +} diff --git a/backend/hls/src/segment_repair.rs b/backend/hls/src/segment_repair.rs index 8ce7d9be3..d94da19ef 100644 --- a/backend/hls/src/segment_repair.rs +++ b/backend/hls/src/segment_repair.rs @@ -38,6 +38,7 @@ struct HlsSelectedRepairCandidate<'a> { runtime: Arc, candidate: HlsRepairWindowCandidateKey, prewarm_guard: Option<&'a HlsRepairPrewarmGuard>, + write_budget: Option>, } #[derive(Debug, Clone, Eq, PartialEq, Hash)] @@ -188,6 +189,8 @@ struct RepairRemuxDroppedStream { mod tests; mod diagnostics; +mod disk_output; +pub(crate) use disk_output::{run_reserved_remux, RepairDiskBudget}; mod manager; mod probe; mod processing; diff --git a/backend/hls/src/segment_repair/disk_output.rs b/backend/hls/src/segment_repair/disk_output.rs new file mode 100644 index 000000000..5814e37a6 --- /dev/null +++ b/backend/hls/src/segment_repair/disk_output.rs @@ -0,0 +1,137 @@ +//! Streams postprocessor output through admission before each filesystem write. +use super::{HlsPostProcessingDeadline, HlsSegmentCache, ProxySessionId, SegmentCacheKey, StagedCacheObject}; +use std::{io, sync::Arc}; +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncWriteExt}, + process::Command, + time::timeout, +}; + +/// Peak ownership and still-live staging bytes for streamed repair output. +#[derive(Clone, Copy)] +pub(crate) struct RepairDiskBudget<'a> { + pub(crate) cache: &'a HlsSegmentCache, + pub(crate) reservation: &'a Arc, + pub(crate) session: &'a ProxySessionId, + pub(crate) retained_staging_bytes: u64, +} + +async fn copy_reserved_output( + reader: &mut R, + file: &mut File, + budget: RepairDiskBudget<'_>, +) -> io::Result<()> { + let key = SegmentCacheKey::new(budget.session.clone(), 0, "ts"); + let mut size = 0u64; + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + break; + } + size = size + .checked_add(u64::try_from(read).map_err(io::Error::other)?) + .ok_or_else(|| io::Error::other("postprocessor output overflow"))?; + budget + .cache + .ensure_write_budget_available(&key, budget.reservation, size.saturating_add(budget.retained_staging_bytes)) + .await?; + file.write_all(&buffer[..read]).await?; + } + file.flush().await?; + Ok(()) +} + +/// Runs an MPEG-TS writer with stdout backpressure so it cannot grow a disk +/// file past its peak reservation between two capacity checks. +pub(crate) async fn run_reserved_remux( + args: &[&str], + output: &StagedCacheObject, + budget: RepairDiskBudget<'_>, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let remaining = deadline.remaining().ok_or_else(|| "timeout".to_string())?; + timeout(remaining, async { + let writer = output.track_writer(File::create(&output.path).await?); + let mut writer = writer.lock().await; + let file = writer.as_mut().ok_or_else(|| io::Error::other("postprocessor output already closed"))?; + let mut command = Command::new("ffmpeg"); + let mut child = command + .args(args) + .arg("pipe:1") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn()?; + let mut stdout = child.stdout.take().ok_or_else(|| io::Error::other("postprocessor stdout unavailable"))?; + copy_reserved_output(&mut stdout, file, budget).await?; + writer.take(); + if !child.wait().await?.success() { + return Err(io::Error::other("postprocessor failed")); + } + Ok::<(), io::Error>(()) + }) + .await + .map_err(|_| "timeout".to_string())? + .map_err(|error| format!("reserved_postprocess_error:{:?}", error.kind())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn postprocessor_output_extends_its_peak_before_writing_or_aborts_without_overshoot() -> io::Result<()> { + for output_size in [100usize, 128] { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(120, 120); + let session = ProxySessionId("postprocess".into()); + let key = SegmentCacheKey::new(session.clone(), 0, "ts"); + let reservation = Arc::new(cache.reserve_revision_peak(&key, 24).await?); + let raw = cache + .stage_temp_with_budget( + &key, + &[0; 8][..], + tokio::time::Instant::now() + std::time::Duration::from_secs(2), + Some(reservation.clone()), + ) + .await?; + let path = raw.path.with_extension("ts.tmp.output"); + let mut output = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(reservation.clone()))?; + let mut file = File::create(&path).await?; + let bytes = vec![7u8; output_size]; + let result = copy_reserved_output( + &mut bytes.as_slice(), + &mut file, + RepairDiskBudget { + cache: &cache, + reservation: &reservation, + session: &session, + retained_staging_bytes: raw.size, + }, + ) + .await; + drop(file); + output.size = tokio::fs::metadata(&path).await?.len(); + assert!(output.size + raw.size <= 120); + if output_size == 100 { + result?; + assert_eq!(output.size, 100); + assert!(cache.remaining_revision_bytes(&session) >= 108); + cache.remove_staged(raw).await?; + cache.commit_staged(&key, output).await?; + assert_eq!(cache.capacity_usage(&session).await?.session_bytes, 100); + } else { + assert!(result.is_err()); + assert_eq!(output.size, 0); + cache.remove_staged(raw).await?; + cache.remove_staged(output).await?; + } + drop(reservation); + assert_eq!(cache.remaining_revision_bytes(&session), 0); + } + Ok(()) + } +} diff --git a/backend/hls/src/segment_repair/manager.rs b/backend/hls/src/segment_repair/manager.rs index f541f30a2..5151d9670 100644 --- a/backend/hls/src/segment_repair/manager.rs +++ b/backend/hls/src/segment_repair/manager.rs @@ -1,11 +1,11 @@ use super::{ - ffmpeg_identity_version, probe::adopt_repair_output, registry::repair_object_metadata_key, - safe_hls_access_lease_id, safe_proxy_session_id, sha256_file, window::RepairStatus, CachedSegmentMetadata, - HlsAccessLeaseId, HlsCacheObjectKey, HlsCorruptSegmentWatchdogManager, HlsPostProcessingDeadline, - HlsRepairObjectMetadata, HlsRepairObjectMetadataKey, HlsRepairPrewarmGuard, HlsRepairRenderedObjectId, - HlsRepairWindowRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, - HlsSegmentRepairRuntime, HlsSegmentRepairStats, HlsSelectedRepairCandidate, ProxySessionId, RepairIdentity, - SegmentRepairMetadata, COMMAND_VERSION, REPAIR_METADATA_MAX_ENTRIES, REPAIR_OBJECT_METADATA_MAX_ENTRIES, + ffmpeg_identity_version, registry::repair_object_metadata_key, safe_hls_access_lease_id, safe_proxy_session_id, + sha256_file, window::RepairStatus, CachedSegmentMetadata, HlsAccessLeaseId, HlsCacheObjectKey, + HlsCorruptSegmentWatchdogManager, HlsPostProcessingDeadline, HlsRepairObjectMetadata, HlsRepairObjectMetadataKey, + HlsRepairPrewarmGuard, HlsRepairRenderedObjectId, HlsRepairWindowRegistry, HlsSegmentCache, + HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairRuntime, HlsSegmentRepairStats, + HlsSelectedRepairCandidate, ProxySessionId, RepairIdentity, SegmentRepairMetadata, COMMAND_VERSION, + REPAIR_METADATA_MAX_ENTRIES, REPAIR_OBJECT_METADATA_MAX_ENTRIES, }; use arc_swap::ArcSwap; use log::debug; @@ -16,7 +16,6 @@ use std::{ sync::Arc, }; use tokio::{ - fs, io::AsyncRead, sync::{Mutex, RwLock, Semaphore}, }; @@ -157,7 +156,13 @@ impl HlsSegmentRepairManager { let segment_cache = Arc::clone(&segment_cache); let guard = guard.clone(); tokio::spawn(async move { - let selection = HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: guard.as_ref() }; + let selection = HlsSelectedRepairCandidate { + mode, + runtime, + candidate, + prewarm_guard: guard.as_ref(), + write_budget: None, + }; if let Err(err) = manager.repair_ready_cache_hit_with_candidate(&segment_cache, &key, context, selection).await { @@ -210,7 +215,23 @@ impl HlsSegmentRepairManager { K: HlsCacheObjectKey, R: AsyncRead + Unpin, { - let raw = segment_cache.stage_temp_with_deadline(key, reader, deadline).await?; + self.commit_origin_response_with_budget(segment_cache, key, reader, deadline, context, None).await + } + + pub(crate) async fn commit_origin_response_with_budget( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + reader: R, + deadline: tokio::time::Instant, + context: HlsSegmentRepairObjectContext, + budget: Option>, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let raw = segment_cache.stage_temp_with_budget(key, reader, deadline, budget).await?; let runtime = self.runtime.load_full(); if !runtime.postprocessing_enabled() { return segment_cache.commit_staged(key, raw).await; @@ -229,6 +250,16 @@ impl HlsSegmentRepairManager { where K: HlsCacheObjectKey, { + self.repair_ready_cache_hit_with_budget(segment_cache, key, context, None).await + } + + pub(crate) async fn repair_ready_cache_hit_with_budget( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + context: HlsSegmentRepairObjectContext, + write_budget: Option>, + ) -> io::Result> { let Some((mode, runtime, candidate)) = self.try_select_or_join_candidate(&context).await else { return Ok(None); }; @@ -236,7 +267,7 @@ impl HlsSegmentRepairManager { segment_cache, key, context, - HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None }, + HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None, write_budget }, ) .await } @@ -252,7 +283,7 @@ impl HlsSegmentRepairManager { where K: HlsCacheObjectKey, { - let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard } = selection; + let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard, write_budget } = selection; let Some(metadata) = segment_cache.metadata(key).await? else { return Ok(None); }; @@ -296,10 +327,19 @@ impl HlsSegmentRepairManager { Ok(None) } else { let deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); - if let Some(fixed_path) = self + if let Some(staged) = self .repair_file( - ¤t_metadata.path, - current_metadata.size, + super::processing::RepairFileSource { + cache: segment_cache, + path: ¤t_metadata.path, + size: current_metadata.size, + disk_budget: write_budget.as_ref().map(|reservation| super::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: 0, + }), + }, &identity, &context, runtime.clone(), @@ -317,11 +357,9 @@ impl HlsSegmentRepairManager { None => None, }; if !candidate_current || latest_hash.as_deref() != Some(current_hash.as_str()) { - let _ = fs::remove_file(&fixed_path).await; + segment_cache.remove_staged(staged).await?; Ok(None) } else { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - let staged = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; let committed = segment_cache.commit_staged(key, staged).await?; self.record_metadata( identity.clone(), diff --git a/backend/hls/src/segment_repair/probe.rs b/backend/hls/src/segment_repair/probe.rs index 83eccb9df..b1baf9a01 100644 --- a/backend/hls/src/segment_repair/probe.rs +++ b/backend/hls/src/segment_repair/probe.rs @@ -1,9 +1,9 @@ use super::{ diagnostics::{decide_repair, SegmentProbeStreamType}, window::{RepairStatus, RepairVideoCodec}, - HlsSegmentCache, RepairRemuxDroppedStream, RepairRemuxStreamSelection, SegmentProbe, SegmentProbeStream, - StagedCacheObject, WarningCounters, + RepairRemuxDroppedStream, RepairRemuxStreamSelection, SegmentProbe, SegmentProbeStream, WarningCounters, }; +#[cfg(test)] use log::warn; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -12,7 +12,9 @@ use std::{ fmt, io, path::{Path, PathBuf}, }; -use tokio::{fs, fs::File, io::AsyncReadExt}; +#[cfg(test)] +use tokio::fs; +use tokio::{fs::File, io::AsyncReadExt}; use tuliprox_core::model::HlsSegmentRepairConfig; pub(super) fn parse_probe(json: &str, warnings: WarningCounters) -> Result { @@ -240,6 +242,7 @@ pub(super) fn repair_output_path(raw_path: &Path) -> PathBuf { raw_path.with_file_name(format!("{file_name}.repair.tmp.{suffix:016x}")) } +#[cfg(test)] pub(super) async fn cleanup_repair_output(path: &Path) { match fs::remove_file(path).await { Ok(()) => {} @@ -248,20 +251,6 @@ pub(super) async fn cleanup_repair_output(path: &Path) { } } -pub(super) async fn adopt_repair_output( - segment_cache: &HlsSegmentCache, - path: PathBuf, - size: u64, -) -> io::Result { - match segment_cache.adopt_staged_file(path.clone(), size) { - Ok(staged) => Ok(staged), - Err(error) => { - cleanup_repair_output(&path).await; - Err(error) - } - } -} - impl fmt::Display for RepairStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -275,3 +264,18 @@ impl fmt::Display for RepairStatus { } } } + +#[cfg(test)] +pub(super) async fn adopt_repair_output( + segment_cache: &super::HlsSegmentCache, + path: PathBuf, + size: u64, +) -> io::Result { + match segment_cache.adopt_staged_file(path.clone(), size) { + Ok(staged) => Ok(staged), + Err(error) => { + cleanup_repair_output(&path).await; + Err(error) + } + } +} diff --git a/backend/hls/src/segment_repair/processing.rs b/backend/hls/src/segment_repair/processing.rs index 9106d4934..199c5c005 100644 --- a/backend/hls/src/segment_repair/processing.rs +++ b/backend/hls/src/segment_repair/processing.rs @@ -1,10 +1,7 @@ use super::{ diagnostics::{debug_repair_analysis, debug_repair_event, debug_repair_stream_dropped, decide_repair}, ffmpeg_identity_version, parse_ffmpeg_warnings, - probe::{ - adopt_repair_output, cleanup_repair_output, detect_video_codec, parse_probe, repair_output_path, - select_repair_remux_streams, validate_repair, - }, + probe::{detect_video_codec, parse_probe, repair_output_path, select_repair_remux_streams, validate_repair}, registry::repair_object_metadata_key, sha256_file, window::{RepairCandidateSelection, RepairStatus}, @@ -15,15 +12,19 @@ use super::{ }; use log::{debug, warn}; use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; -use std::{ - io, - path::{Path, PathBuf}, - sync::Arc, - time::Instant, -}; +use std::{io, path::Path, sync::Arc, time::Instant}; use tokio::{fs, process::Command, time::timeout}; +/// Files and peak ownership used for one postprocessing attempt. +pub(super) struct RepairFileSource<'a> { + pub(super) cache: &'a HlsSegmentCache, + pub(super) path: &'a Path, + pub(super) size: u64, + pub(super) disk_budget: Option>, +} + impl HlsSegmentRepairManager { + #[allow(clippy::too_many_lines)] pub(super) async fn process_staged_and_commit( &self, segment_cache: &HlsSegmentCache, @@ -91,14 +92,27 @@ impl HlsSegmentRepairManager { Ok(committed) } else { let raw_size = raw.size; - if let Some(fixed_path) = - self.repair_file(&raw.path, raw.size, &identity, &context, runtime.clone(), &deadline).await? + if let Some(fixed) = self + .repair_file( + RepairFileSource { + cache: segment_cache, + path: &raw.path, + size: raw.size, + disk_budget: raw.write_budget.as_ref().map(|reservation| super::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: raw.size, + }), + }, + &identity, + &context, + runtime.clone(), + &deadline, + ) + .await? { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - if let Err(error) = segment_cache.remove_staged(raw.clone()).await { - warn!("HLS repair raw staging cleanup deferred: error_kind={:?}", error.kind()); - } - let fixed = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; + segment_cache.remove_staged(raw.clone()).await?; let committed = segment_cache.commit_staged(key, fixed).await?; self.record_metadata(identity.clone(), RepairStatus::Fixed, raw.size, committed.size, None).await; let committed_hash = sha256_file(&committed.path).await?; @@ -201,13 +215,13 @@ impl HlsSegmentRepairManager { #[allow(clippy::too_many_lines)] pub(super) async fn repair_file( &self, - raw_path: &Path, - raw_size: u64, + source: RepairFileSource<'_>, identity: &RepairIdentity, context: &HlsSegmentRepairObjectContext, runtime: Arc, deadline: &HlsPostProcessingDeadline, - ) -> io::Result> { + ) -> io::Result> { + let RepairFileSource { cache: segment_cache, path: raw_path, size: raw_size, disk_budget } = source; let _permit = match &runtime.semaphore { Some(semaphore) => { let Some(remaining) = deadline.remaining() else { @@ -297,11 +311,30 @@ impl HlsSegmentRepairManager { identity.repair_mode.as_log_value(), executed_level.as_log_value() ); - let remux = run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline).await; + // A ready-cache prewarm has no startup owner. Give its output the same + // admission and cancellation guarantees as a downloaded staging file. + let owned_budget; + let disk_budget = if let Some(budget) = disk_budget { + budget + } else { + let key = crate::SegmentCacheKey::new(context.proxy_session_id.clone(), 0, "ts"); + owned_budget = Arc::new(segment_cache.reserve_revision_peak_reclaiming(&key, raw_size).await?); + super::RepairDiskBudget { + cache: segment_cache, + reservation: &owned_budget, + session: &context.proxy_session_id, + retained_staging_bytes: 0, + } + }; + let mut output = segment_cache + .adopt_staged_file(fixed_path.clone(), 0)? + .with_write_budget(Some(Arc::clone(disk_budget.reservation)))?; + let remux = + run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline, disk_budget, &output).await; if let Err(reason) = remux { debug_repair_event(context, executed_level, "remux failed", Some(&reason)); let status = if reason == "timeout" { RepairStatus::Timeout } else { RepairStatus::RemuxFailed }; - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata(identity.clone(), status, raw_size, raw_size, Some(reason)).await; return Ok(None); } @@ -309,7 +342,7 @@ impl HlsSegmentRepairManager { Ok(scan) => scan, Err(reason) => { debug_repair_event(context, executed_level, "validation probe failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata( identity.clone(), RepairStatus::ValidationFailed, @@ -325,7 +358,7 @@ impl HlsSegmentRepairManager { validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); if let Err(reason) = validation { debug_repair_event(context, executed_level, "validation failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; + segment_cache.remove_staged(output).await?; self.record_metadata(identity.clone(), RepairStatus::ValidationFailed, raw_size, raw_size, Some(reason)) .await; return Ok(None); @@ -339,7 +372,8 @@ impl HlsSegmentRepairManager { executed_level.as_log_value(), started.elapsed().as_millis() ); - Ok(Some(fixed_path)) + output.size = fs::metadata(&fixed_path).await?.len(); + Ok(Some(output)) } } @@ -390,6 +424,8 @@ async fn run_remux( mode: HlsSegmentRepairMode, stream_selection: &RepairRemuxStreamSelection, deadline: &HlsPostProcessingDeadline, + disk_budget: super::RepairDiskBudget<'_>, + staged: &StagedCacheObject, ) -> Result<(), String> { let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; @@ -417,7 +453,7 @@ async fn run_remux( .map(ToOwned::to_owned), ); let args = args.iter().map(String::as_str).collect::>(); - run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) + super::run_reserved_remux(&args[..args.len() - 1], staged, disk_budget, deadline).await } pub(crate) async fn run_command_with_deadline( diff --git a/backend/hls/src/segment_repair/tests/transport.rs b/backend/hls/src/segment_repair/tests/transport.rs index 199636976..2a723d0d6 100644 --- a/backend/hls/src/segment_repair/tests/transport.rs +++ b/backend/hls/src/segment_repair/tests/transport.rs @@ -16,6 +16,7 @@ fn repair_prewarm_candidates_include_only_ready_clear_origin_ts_media() { host_local_index: 0, }, proxy_seq, + media_start_ms: 0, duration_ms: 6_000, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), diff --git a/backend/hls/src/segment_revision.rs b/backend/hls/src/segment_revision.rs index 85c6d49e7..7a624b599 100644 --- a/backend/hls/src/segment_revision.rs +++ b/backend/hls/src/segment_revision.rs @@ -279,12 +279,14 @@ impl SegmentRevisionStore { Ok(()) } - pub fn retire_unpinned(&self) -> io::Result>> { + pub fn retire_unpinned(&self) -> io::Result>> { self.retire_unpinned_limit(128) } + + pub(crate) fn retire_unpinned_limit(&self, limit: usize) -> io::Result>> { let mut entries = self.entries.lock().map_err(|_| io::Error::other("revision registry poisoned"))?; let ids: Vec<_> = entries .iter() .filter_map(|(id, revision)| (revision.pins.load(Ordering::Acquire) == 0).then_some(*id)) - .take(128) + .take(limit) .collect(); Ok(ids.into_iter().filter_map(|id| entries.remove(&id)).collect()) } diff --git a/backend/hls/src/segment_watchdog.rs b/backend/hls/src/segment_watchdog.rs index f589532d6..ce3d0380c 100644 --- a/backend/hls/src/segment_watchdog.rs +++ b/backend/hls/src/segment_watchdog.rs @@ -275,8 +275,16 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } let fixed_path = watchdog_output_path(&raw.path); - if let Err(reason) = sanitize_corrupt_segment(&raw.path, &fixed_path, deadline).await { - cleanup_watchdog_output(&fixed_path).await; + let mut fixed = + segment_cache.adopt_staged_file(fixed_path.clone(), 0)?.with_write_budget(raw.write_budget.clone())?; + let disk_budget = raw.write_budget.as_ref().map(|reservation| crate::segment_repair::RepairDiskBudget { + cache: segment_cache, + reservation, + session: &context.proxy_session_id, + retained_staging_bytes: raw.size, + }); + if let Err(reason) = sanitize_corrupt_segment(&raw.path, &fixed_path, deadline, &fixed, disk_budget).await { + segment_cache.remove_staged(fixed).await?; let status = if reason == "timeout" { HlsCorruptSegmentWatchdogStatus::Timeout } else { @@ -289,7 +297,7 @@ impl HlsCorruptSegmentWatchdogManager { Ok(metadata) => metadata.len(), Err(err) => { let reason = format!("metadata_failed:{err}"); - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -304,7 +312,7 @@ impl HlsCorruptSegmentWatchdogManager { let fixed_detect = match detect_packet_corrupt(&fixed_path, deadline).await { Ok(count) => count, Err(reason) if reason == "timeout" => { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::Timeout, @@ -316,7 +324,7 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } Err(reason) => { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -339,7 +347,7 @@ impl HlsCorruptSegmentWatchdogManager { ) .await; if let Err(reason) = validation { - cleanup_watchdog_output(&fixed_path).await; + segment_cache.remove_staged(fixed).await?; self.record_metadata( identity, HlsCorruptSegmentWatchdogStatus::ValidationFailed { reason: reason.clone() }, @@ -351,7 +359,8 @@ impl HlsCorruptSegmentWatchdogManager { return segment_cache.commit_staged(key, raw).await; } let raw_size = raw.size; - let committed = match commit_watchdog_output_or_raw(segment_cache, key, raw, fixed_path, fixed_size).await? { + fixed.size = fixed_size; + let committed = match commit_watchdog_staged_or_raw(segment_cache, key, raw, Ok(fixed)).await? { HlsWatchdogCommitOutcome::Sanitized(committed) => committed, HlsWatchdogCommitOutcome::RawFallback(committed) => return Ok(committed), }; @@ -485,40 +494,40 @@ async fn sanitize_corrupt_segment( input_path: &Path, output_path: &Path, deadline: &HlsPostProcessingDeadline, + staged: &StagedCacheObject, + disk_budget: Option>, ) -> Result<(), String> { let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; - run_command_with_deadline( - "ffmpeg", - &[ - "-hide_banner", - "-nostdin", - "-y", - "-fflags", - "+discardcorrupt", - "-copyts", - "-i", - input, - "-map", - "0", - "-c", - "copy", - "-mpegts_flags", - "+resend_headers", - "-mpegts_copyts", - "1", - "-muxpreload", - "0", - "-muxdelay", - "0", - "-f", - "mpegts", - output, - ], - deadline, - ) - .await - .map(|_| ()) + let args = [ + "-hide_banner", + "-nostdin", + "-y", + "-fflags", + "+discardcorrupt", + "-copyts", + "-i", + input, + "-map", + "0", + "-c", + "copy", + "-mpegts_flags", + "+resend_headers", + "-mpegts_copyts", + "1", + "-muxpreload", + "0", + "-muxdelay", + "0", + "-f", + "mpegts", + output, + ]; + if let Some(budget) = disk_budget { + return crate::segment_repair::run_reserved_remux(&args[..args.len() - 1], staged, budget, deadline).await; + } + run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) } async fn validate_sanitized_segment( @@ -661,6 +670,7 @@ fn watchdog_output_path(input_path: &Path) -> PathBuf { PathBuf::from(value) } +#[cfg(test)] async fn cleanup_watchdog_output(path: &Path) { match fs::remove_file(path).await { Ok(()) => {} @@ -669,6 +679,7 @@ async fn cleanup_watchdog_output(path: &Path) { } } +#[cfg(test)] async fn adopt_watchdog_output( segment_cache: &HlsSegmentCache, path: PathBuf, @@ -683,6 +694,7 @@ async fn adopt_watchdog_output( } } +#[cfg(test)] async fn commit_watchdog_output_or_raw( segment_cache: &HlsSegmentCache, key: &K, @@ -693,23 +705,33 @@ async fn commit_watchdog_output_or_raw( where K: HlsCacheObjectKey, { - let fixed = match adopt_watchdog_output(segment_cache, fixed_path, fixed_size).await { + let fixed = adopt_watchdog_output(segment_cache, fixed_path, fixed_size).await; + commit_watchdog_staged_or_raw(segment_cache, key, raw, fixed).await +} + +async fn commit_watchdog_staged_or_raw( + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + fixed: io::Result, +) -> io::Result { + let fixed = match fixed.and_then(|fixed| fixed.with_write_budget(raw.write_budget.clone())) { Ok(fixed) => fixed, Err(error) => { warn!("HLS watchdog fixed staging adoption failed; committing raw fallback: error_kind={:?}", error.kind()); return segment_cache.commit_staged(key, raw).await.map(HlsWatchdogCommitOutcome::RawFallback); } }; + let fixed_cleanup = fixed.clone(); let committed = match segment_cache.commit_staged(key, fixed).await { Ok(committed) => committed, Err(error) => { warn!("HLS watchdog fixed staging commit failed; committing raw fallback: error_kind={:?}", error.kind()); + segment_cache.remove_staged(fixed_cleanup).await?; return segment_cache.commit_staged(key, raw).await.map(HlsWatchdogCommitOutcome::RawFallback); } }; - if let Err(error) = segment_cache.remove_staged(raw).await { - warn!("HLS watchdog raw staging cleanup deferred: error_kind={:?}", error.kind()); - } + segment_cache.remove_staged(raw).await?; Ok(HlsWatchdogCommitOutcome::Sanitized(committed)) } @@ -763,8 +785,8 @@ fn debug_watchdog_event( #[cfg(test)] mod tests { use super::{ - commit_watchdog_output_or_raw, count_packet_corrupt_events, parse_packet_corrupt_dts, - unsupported_sanitize_extension, HlsWatchdogCommitOutcome, + commit_watchdog_output_or_raw, commit_watchdog_staged_or_raw, count_packet_corrupt_events, + parse_packet_corrupt_dts, unsupported_sanitize_extension, HlsWatchdogCommitOutcome, }; use crate::{HlsSegmentCache, ProxySessionId, SegmentCacheKey}; use std::time::Duration; @@ -849,4 +871,33 @@ Last message repeated 3 times assert_eq!(tokio::fs::read(committed.path).await.expect("committed fallback reads"), b"raw"); assert!(!fixed_path.exists()); } + + #[tokio::test] + async fn failed_raw_staging_cleanup_aborts_postprocessing_before_further_publication() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(40, 40); + let key = SegmentCacheKey::new(ProxySessionId("cleanup-failure".into()), 1, "ts"); + let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?); + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + let raw = cache.stage_temp_with_budget(&key, &[1; 10][..], deadline, Some(budget.clone())).await?; + let fixed = cache.stage_temp_with_budget(&key, &[2; 20][..], deadline, Some(budget.clone())).await?; + cache.fail_next_cache_deletes(raw.path.clone(), 1); + + let error = commit_watchdog_staged_or_raw(&cache, &key, raw.clone(), Ok(fixed)) + .await + .err() + .expect("failed temp removal must stop the owning startup transaction"); + + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert_eq!(tokio::fs::read(&raw.path).await?, [1; 10]); + assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10); + assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 20]); + cache.remove_staged(raw).await?; + drop(budget); + assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0); + assert!(!cache.has_active_temp_files()); + Ok(()) + } } diff --git a/backend/hls/src/session.rs b/backend/hls/src/session.rs index 14beb4aa3..5e0211392 100644 --- a/backend/hls/src/session.rs +++ b/backend/hls/src/session.rs @@ -120,6 +120,7 @@ pub struct HlsSessionOriginControl { host_local_highwater_observation_order: u64, pub acceptance_episode: Option, pub recovery_samples: HlsBoundedRecoverySamples, + pub(crate) qualification: crate::short_manifest::HlsOriginQualification, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -146,6 +147,7 @@ impl Default for HlsSessionOriginControl { host_local_highwater_observation_order: 0, acceptance_episode: None, recovery_samples: HlsBoundedRecoverySamples::default(), + qualification: crate::short_manifest::HlsOriginQualification::default(), } } } @@ -155,6 +157,18 @@ impl HlsSessionOriginControl { pub fn record_origin_response(&mut self, now_ms: u64) { self.last_origin_response_at_ms = Some(now_ms); } + /// Includes publication lateness even when the most recent HTTP request has not failed yet. + pub(crate) fn path_is_degraded_at(&self, now_ms: u64) -> bool { + self.path_condition.is_degraded() + || self.last_media_progress_at_ms.zip(self.target_duration_snapshot_ms).is_some_and( + |(last_progress_ms, target_duration_ms)| { + target_duration_ms > 0 + && now_ms.saturating_sub(last_progress_ms) + >= crate::origin_progress::publication_late_after_ms(target_duration_ms) + }, + ) + } + pub fn record_media_progress(&mut self, now_ms: u64, target_duration_ms: u64) { self.last_origin_response_at_ms = Some(now_ms); self.last_media_progress_at_ms = Some(now_ms); @@ -253,6 +267,9 @@ pub struct HlsSession { pub origin_account_binding: Option, pub origin_account_io_lease: Option>, pub origin_account_rebind: HlsOriginAccountRebindState, + pub origin_account_prepare_lock: Arc>, + pub origin_account_runtime: Option>, + pub origin_account_manifest_pending: Option, pub effective_origin_acquire_policy: Option, pub mode: HlsSessionMode, pub transient: TransientPassthroughState, @@ -286,6 +303,8 @@ pub struct HlsSession { pub origin_epoch_sequence_base: Option, pub origin_seq_highwater: Option, pub proxy_next_seq: Option, + /// End of all appended media, retained even when every old cache entry is collected. + pub next_media_start_ms: u64, pub origin_to_proxy: HashMap, pub published_resource_history: HlsPublishedResourceHistory, pub discontinuity_sequence: u64, @@ -330,6 +349,9 @@ impl HlsSession { origin_account_binding: None, origin_account_io_lease: None, origin_account_rebind: HlsOriginAccountRebindState::default(), + origin_account_prepare_lock: Arc::new(tokio::sync::Mutex::new(())), + origin_account_runtime: None, + origin_account_manifest_pending: None, effective_origin_acquire_policy: None, mode: HlsSessionMode::NormalCacheTimeline, transient: TransientPassthroughState::default(), @@ -360,6 +382,7 @@ impl HlsSession { origin_epoch_sequence_base: None, origin_seq_highwater: None, proxy_next_seq: None, + next_media_start_ms: 0, origin_to_proxy: HashMap::new(), published_resource_history: HlsPublishedResourceHistory::default(), discontinuity_sequence: 0, @@ -466,6 +489,43 @@ impl HlsSession { pub fn is_gc_marked_for_removal(&self) -> bool { self.gc_marked_for_removal } + /// Invalidates account-dependent work while preserving published media and host/content anchors. + pub fn handoff_origin_account(&mut self, binding: HlsOriginAccountBinding) { + self.invalidate_queued_origin_work(); + self.origin_account_io_lease = None; + self.origin_provider_session_headers.clear(); + self.origin_control.manifest_origin_binding = None; + self.origin_control.acceptance_episode = None; + self.origin_control.acceptance_generation.0 = self.origin_control.acceptance_generation.0.saturating_add(1); + self.origin_control.qualification = crate::short_manifest::HlsOriginQualification::default(); + self.origin_refresh = OriginRefreshState::default(); + self.origin_account_manifest_pending = Some(binding.generation); + self.transient.invalidate_origin_resources(); + for segment in self.segments.values_mut() { + if !matches!(segment.status, SegmentCacheStatus::Ready { .. }) { + segment.origin_fetch_ref = None; + if !matches!(segment.status, SegmentCacheStatus::Fetching { .. }) { + segment.status = SegmentCacheStatus::Discovered; + } + } + } + for map in self.maps.values_mut() { + if !matches!(map.status, MapCacheStatus::Ready { .. }) { + map.origin_fetch_ref = None; + if !matches!(map.status, MapCacheStatus::Fetching { .. }) { + map.status = MapCacheStatus::Discovered; + } + } + } + self.replace_origin_account_binding(Some(binding)); + } + + pub(crate) fn origin_handoff_ready_after(&self, generation: u64) -> bool { + self.activity.origin_work_generation != generation + && self.origin_account_manifest_pending.is_none() + && !self.is_gc_marked_for_removal() + } + pub fn replace_origin_account_binding(&mut self, binding: Option) { let binding_changed = match (&self.origin_account_binding, &binding) { (Some(current), Some(next)) => { @@ -778,6 +838,15 @@ impl HlsSession { }; } + /// Provider lease kind retained across account replacement and subsequent origin I/O. + pub const fn origin_playback_kind(&self) -> tuliprox_core::model::PlaybackKind { + if self.key.archive_reference.is_some() { + tuliprox_core::model::PlaybackKind::Catchup + } else { + tuliprox_core::model::PlaybackKind::LiveHls + } + } + pub fn start_origin_work(&mut self) -> u64 { self.activity.active_origin_work_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); self.activity.origin_work_generation @@ -828,6 +897,9 @@ impl fmt::Debug for HlsSession { .field("origin_account_binding", &self.origin_account_binding) .field("origin_account_io_lease", &self.origin_account_io_lease) .field("origin_account_rebind", &self.origin_account_rebind) + .field("origin_account_prepare_lock", &self.origin_account_prepare_lock) + .field("origin_account_runtime", &self.origin_account_runtime) + .field("origin_account_manifest_pending", &self.origin_account_manifest_pending) .field("effective_origin_acquire_policy", &self.effective_origin_acquire_policy) .field("mode", &self.mode) .field("transient", &self.transient) @@ -858,6 +930,7 @@ impl fmt::Debug for HlsSession { .field("origin_epoch_sequence_base", &self.origin_epoch_sequence_base) .field("origin_seq_highwater", &self.origin_seq_highwater) .field("proxy_next_seq", &self.proxy_next_seq) + .field("next_media_start_ms", &self.next_media_start_ms) .field("origin_to_proxy_len", &self.origin_to_proxy.len()) .field("published_resource_history", &self.published_resource_history.generation()) .field("discontinuity_sequence", &self.discontinuity_sequence) @@ -934,6 +1007,7 @@ mod tests { host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), }, proxy_seq, + media_start_ms: proxy_seq * duration_ms, duration_ms, proxy_file_ext: "ts".to_string(), content_type: "video/mp2t".to_string(), @@ -1713,4 +1787,44 @@ mod tests { assert_eq!(fallback.connection_kind, ConnectionKind::Normal); assert_eq!(fallback.priority, 0); } + #[test] + fn account_handoff_invalidates_work_and_recovery_but_preserves_timeline_and_archive() { + let mut session = recovery_eligible_session(); + session.key = session.key.with_archive_reference(123).with_archive_identity("archive-identity"); + session.origin_source = crate::HlsOriginSource::from_session_key(&session.key); + let source = session.origin_source.clone(); + let entry = ready_segment(&session, 10, 0, 4_000, 32); + session.segments.insert(10, entry); + session.activity.active_access_lease_count = 2; + let key = session.key.clone(); + let proxy_id = session.proxy_session_id.clone(); + let rendered = session.last_rendered_manifest.clone(); + let pinned_host = session.origin_control.pinned_host.clone(); + session + .origin_provider_session_headers + .insert(axum::http::header::COOKIE, axum::http::HeaderValue::from_static("old=secret")); + let old_work = session.start_origin_work(); + let binding = crate::origin::HlsOriginAccountBinding::rebound( + Arc::from("input"), + Arc::from("replacement"), + "owner".to_string(), + 3, + 10, + ); + session.handoff_origin_account(binding); + assert!(!session.finish_origin_work(old_work)); + assert_eq!(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), 0); + assert_eq!((&session.key, &session.proxy_session_id), (&key, &proxy_id)); + assert_eq!(session.last_rendered_manifest, rendered); + assert_eq!(session.origin_control.pinned_host, pinned_host); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.activity.active_access_lease_count, 2); + assert_eq!(session.origin_source, source); + assert_eq!(session.origin_playback_kind(), tuliprox_core::model::PlaybackKind::Catchup); + assert_eq!(session.origin_account_manifest_pending, Some(3)); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); + assert!(session.origin_provider_session_headers.is_empty()); + assert!(matches!(session.segments[&10].status, SegmentCacheStatus::Ready { content_length: 32, .. })); + } } diff --git a/backend/hls/src/short_manifest.rs b/backend/hls/src/short_manifest.rs new file mode 100644 index 000000000..0888f1eec --- /dev/null +++ b/backend/hls/src/short_manifest.rs @@ -0,0 +1,214 @@ +//! Session-owned permission for an origin's unusually short live playlists. + +use crate::{ + manifest_acceptance::HlsManifestAcceptanceGeneration, manifest_origin_binding::HlsManifestOriginBinding, + media_reserve::HlsManifestCommitIdentity, HlsSession, +}; + +/// The validated origin foundation, deliberately independent of window movement and leases. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct HlsOriginQualificationBasis { + epoch: u64, + work_generation: u64, + binding: Option, + host: Option, +} + +impl HlsOriginQualificationBasis { + pub(crate) fn capture(session: &HlsSession) -> Self { + Self { + epoch: session.origin_epoch, + work_generation: session.activity.origin_work_generation, + binding: session.origin_control.manifest_origin_binding.clone(), + host: session.origin_control.pinned_host.clone(), + } + } +} + +/// Window sizes accepted for the current validated origin foundation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HlsQualifiedOriginWindows { + Regular, + ShortAllowed, +} + +/// One shared qualification and its retry receipt; never consumed by a publication. +#[derive(Clone, Debug, Default)] +pub(crate) enum HlsOriginQualification { + #[default] + Unchecked, + Checking { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + target: HlsManifestOriginBinding, + }, + Qualified { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + windows: HlsQualifiedOriginWindows, + /// Last publication when this origin foundation was qualified. + publication_boundary: Option, + }, + RetryPending { + basis: HlsOriginQualificationBasis, + generation: HlsManifestAcceptanceGeneration, + target: HlsManifestOriginBinding, + }, +} + +impl HlsOriginQualification { + pub(crate) fn qualified_windows(&self, basis: &HlsOriginQualificationBasis) -> Option { + match self { + Self::Qualified { basis: current, windows, .. } if current == basis => Some(*windows), + Self::Unchecked | Self::Checking { .. } | Self::Qualified { .. } | Self::RetryPending { .. } => None, + } + } + + pub(crate) fn pending_is_current(&self, basis: &HlsOriginQualificationBasis) -> bool { + match self { + Self::Checking { basis: current, .. } | Self::RetryPending { basis: current, .. } => current == basis, + Self::Unchecked | Self::Qualified { .. } => true, + } + } + + pub(crate) fn retry_target(&self, basis: &HlsOriginQualificationBasis) -> Option<&HlsManifestOriginBinding> { + match self { + Self::RetryPending { basis: current, target, .. } if current == basis => Some(target), + Self::Unchecked | Self::Checking { .. } | Self::Qualified { .. } | Self::RetryPending { .. } => None, + } + } + + pub(crate) fn record_failure(&mut self) { + if let Self::Checking { basis, generation, target } = self { + *self = Self::RetryPending { basis: basis.clone(), generation: *generation, target: target.clone() }; + } + } + + pub(crate) const fn generation(&self) -> Option { + match self { + Self::Unchecked => None, + Self::Checking { generation, .. } + | Self::Qualified { generation, .. } + | Self::RetryPending { generation, .. } => Some(*generation), + } + } +} + +impl HlsSession { + pub(crate) fn origin_qualification_retry_target(&self) -> Option<&HlsManifestOriginBinding> { + // Deterministic conflicts retain their own ordinary-sample path. Only + // changed receipt evidence may authorize a replacement full burst. + if self + .origin_control + .acceptance_episode + .as_ref() + .is_some_and(|episode| episode.deterministic_conflict_receipt().is_some()) + { + return None; + } + self.origin_control.qualification.retry_target(&HlsOriginQualificationBasis::capture(self)) + } + + pub(crate) fn origin_is_qualified(&self) -> bool { + self.origin_control.qualification.qualified_windows(&HlsOriginQualificationBasis::capture(self)).is_some() + } + + pub(crate) fn origin_qualification_awaits_publication(&self) -> bool { + match &self.origin_control.qualification { + HlsOriginQualification::Qualified { basis, publication_boundary, .. } + if basis == &HlsOriginQualificationBasis::capture(self) => + { + self.last_normal_manifest_commit_identity() == *publication_boundary + } + HlsOriginQualification::Unchecked + | HlsOriginQualification::Checking { .. } + | HlsOriginQualification::Qualified { .. } + | HlsOriginQualification::RetryPending { .. } => false, + } + } + + pub(crate) fn allows_short_origin_windows(&self) -> bool { + self.origin_control.qualification.qualified_windows(&HlsOriginQualificationBasis::capture(self)) + == Some(HlsQualifiedOriginWindows::ShortAllowed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HlsSessionKey; + + #[test] + fn permission_survives_progress_but_not_origin_foundation_changes() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "short"), b"secret", 0); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(1), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: None, + }; + for progress in 1..=10 { + session.origin_seq_highwater = Some(progress); + session.origin_control.record_media_progress(progress, 4_000); + assert!(session.allows_short_origin_windows()); + } + session.origin_epoch += 1; + assert!(!session.allows_short_origin_windows()); + session.origin_epoch -= 1; + session.activity.origin_work_generation += 1; + assert!(!session.allows_short_origin_windows()); + session.activity.origin_work_generation -= 1; + session.origin_control.pinned_host = Some("changed.example".to_string()); + assert!(!session.allows_short_origin_windows()); + } + + #[test] + fn unchecked_regular_checking_and_failed_results_do_not_permit_short_publication() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "short"), b"secret", 0); + assert!(!session.allows_short_origin_windows()); + let basis = HlsOriginQualificationBasis::capture(&session); + let generation = HlsManifestAcceptanceGeneration(1); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: basis.clone(), + generation, + windows: HlsQualifiedOriginWindows::Regular, + publication_boundary: None, + }; + assert!(!session.allows_short_origin_windows()); + session.origin_control.qualification = HlsOriginQualification::Checking { + basis, + generation, + target: HlsManifestOriginBinding::new(url::Url::parse("https://origin.example/live.m3u8").unwrap(), None) + .unwrap(), + }; + assert!(!session.allows_short_origin_windows()); + session.origin_control.qualification.record_failure(); + assert!(!session.allows_short_origin_windows()); + } + + #[test] + fn pending_publication_tracks_current_qualification_and_commit_generation() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "revalidation"), b"secret", 0); + let first = session.next_manifest_commit_identity(10).unwrap(); + session.record_normal_manifest_commit_identity(first); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(2), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: Some(first), + }; + assert!(session.origin_qualification_awaits_publication()); + let next = session.next_manifest_commit_identity(10).unwrap(); + session.record_normal_manifest_commit_identity(next); + assert!(!session.origin_qualification_awaits_publication(), "equal timestamps still identify distinct commits"); + session.origin_control.qualification = HlsOriginQualification::Qualified { + basis: HlsOriginQualificationBasis::capture(&session), + generation: HlsManifestAcceptanceGeneration(3), + windows: HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: Some(next), + }; + assert!(session.origin_qualification_awaits_publication()); + session.origin_epoch += 1; + assert!(!session.origin_qualification_awaits_publication()); + } +} diff --git a/backend/hls/src/terminal_commit.rs b/backend/hls/src/terminal_commit.rs index b11f693a7..6b07edb18 100644 --- a/backend/hls/src/terminal_commit.rs +++ b/backend/hls/src/terminal_commit.rs @@ -154,6 +154,7 @@ pub(super) type HlsTerminalCommitAttemptFn = #[derive(Debug, Clone)] pub enum HlsTerminalLeaseDecision { Tail(Arc), + CustomResponse(Arc), Unavailable(HlsTerminalTailCompatibility), UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility), } @@ -303,6 +304,16 @@ impl HlsTerminalCommitCommand { self.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() } + fn submission_deadline_ms(&self, now_ms: u64) -> u64 { + if self.has_independent_commit_deadline() { + // No live media is spliced by this decision. It gets one bounded CAS budget, + // even when the media deadline has already elapsed; repeated submissions cannot renew it. + now_ms.saturating_add(terminal_commit_retry_schedule_budget_ms()) + } else { + self.latest_safe_terminal_commit_at_ms() + } + } + /// Monotonic session/lease evidence used to prevent a late stale request /// from replacing a newer command for the same lease decision generation. fn evidence_version(&self) -> [u64; 7] { @@ -317,8 +328,11 @@ impl HlsTerminalCommitCommand { ] } - fn is_unavailable_after_owner_failure(&self) -> bool { - matches!(&self.decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) + fn has_independent_commit_deadline(&self) -> bool { + matches!( + &self.decision, + HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) + ) } fn may_replace(&self, current: &Self) -> bool { @@ -329,8 +343,8 @@ impl HlsTerminalCommitCommand { return self.session_incarnation > current.session_incarnation; } let evidence_order = self.evidence_version().cmp(¤t.evidence_version()); - if self.is_unavailable_after_owner_failure() - && !current.is_unavailable_after_owner_failure() + if self.has_independent_commit_deadline() + && !current.has_independent_commit_deadline() && evidence_order != std::cmp::Ordering::Less { return true; @@ -359,6 +373,7 @@ impl std::fmt::Debug for HlsTerminalCommitCommand { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let decision_kind = match &self.decision { HlsTerminalLeaseDecision::Tail(_) => "tail", + HlsTerminalLeaseDecision::CustomResponse(_) => "custom-response", HlsTerminalLeaseDecision::Unavailable(_) => "unavailable", HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => "unavailable-after-owner-failure", }; @@ -656,7 +671,7 @@ impl HlsTerminalCommitRetryCoordinator { return HlsTerminalCommitSubmissionDecision::Cancelled; } let key = command.key.clone(); - let incoming_deadline_ms = command.latest_safe_terminal_commit_at_ms(); + let incoming_deadline_ms = command.submission_deadline_ms(now_ms); if let Some(existing) = state.owners.get(&key).map(HlsTerminalCommitExistingOwner::snapshot) { return self.submit_existing(&mut state, command, &key, now_ms, incoming_deadline_ms, existing); } @@ -672,8 +687,21 @@ impl HlsTerminalCommitRetryCoordinator { incoming_deadline_ms: u64, existing: HlsTerminalCommitExistingOwner, ) -> HlsTerminalCommitSubmissionDecision { - let effective_deadline_ms = existing.effective_deadline_ms.min(incoming_deadline_ms); - if !command.may_replace(&existing.command) { + let may_replace = command.may_replace(&existing.command); + let changes_deadline_kind = + command.has_independent_commit_deadline() != existing.command.has_independent_commit_deadline(); + let replaces_splice_with_independent_commit = + changes_deadline_kind && may_replace && command.has_independent_commit_deadline(); + let effective_deadline_ms = if changes_deadline_kind { + if may_replace { + incoming_deadline_ms + } else { + existing.effective_deadline_ms + } + } else { + existing.effective_deadline_ms.min(incoming_deadline_ms) + }; + if !may_replace { let deadline_tightened = effective_deadline_ms < existing.effective_deadline_ms; let tightened_scheduled_at_ms = if deadline_tightened { terminal_commit_scheduled_before_deadline(existing.scheduled_at_ms, now_ms, effective_deadline_ms) @@ -732,7 +760,8 @@ impl HlsTerminalCommitRetryCoordinator { if let Some(owner) = state.owners.get_mut(key) { owner.owner_token = owner_token; owner.command = Arc::clone(&command); - owner.attempts_completed = existing.attempts_completed.max(1); + owner.attempts_completed = + if replaces_splice_with_independent_commit { 1 } else { existing.attempts_completed.max(1) }; owner.effective_deadline_ms = effective_deadline_ms; owner.scheduled_at_ms = now_ms; owner.scheduling_sequence = scheduling_sequence; @@ -1582,7 +1611,7 @@ mod tests { } #[test] - fn unavailable_after_owner_failure_retains_the_exclusive_media_tail_deadline() { + fn unavailable_after_owner_failure_has_a_separate_bounded_cas_deadline() { let mut command = retry_command("failed-closed-unavailable", 10_000, 1, 1); let media_tail_deadline = command.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); @@ -1591,10 +1620,11 @@ mod tests { command.decision = HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); assert_eq!(command.latest_safe_terminal_commit_at_ms(), media_tail_deadline); + assert_eq!(command.submission_deadline_ms(20_000), 20_000 + super::terminal_commit_retry_schedule_budget_ms()); } #[test] - fn failed_media_tail_owner_cannot_be_replaced_at_the_exclusive_deadline() { + fn failed_media_tail_owner_can_finish_unavailable_at_the_exclusive_deadline() { let coordinator = HlsTerminalCommitRetryCoordinator::default(); let command = retry_command("failed-owner-replacement", 10_000, 1, 1); let key = command.key.clone(); @@ -1608,7 +1638,7 @@ mod tests { coordinator.fail_owner(&key, owner_token, HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed); let replacement = HlsTerminalCommitCommand { - key, + key: key.clone(), session, session_incarnation, preparation, @@ -1622,8 +1652,11 @@ mod tests { }; assert!(matches!( coordinator.submit(replacement, failed_at_ms), - HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + HlsTerminalCommitSubmissionDecision::Attempt { .. } )); + let owner = coordinator.owner_snapshot(&key).expect("unavailable CAS owner"); + assert_eq!(owner.effective_deadline_ms, failed_at_ms + super::terminal_commit_retry_schedule_budget_ms()); + assert_eq!(owner.attempts_completed, 1); } #[test] @@ -1897,4 +1930,35 @@ mod tests { assert!(!coordinator.owner_snapshot(&first_key).expect("released worker owner").in_flight); assert!(coordinator.owner_snapshot(&second_key).expect("unrelated client owner").in_flight); } + + #[test] + fn unavailable_cas_budget_is_neither_renewed_by_polls_nor_poisoned_by_stale_media_deadlines() { + let coordinator = HlsTerminalCommitRetryCoordinator::default(); + let mut initial = retry_command("bounded-unavailable", 10_000, 1, 1); + initial.decision = + HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); + let session = Arc::clone(&initial.session); + let key = initial.key.clone(); + assert!(matches!(coordinator.submit(initial, 10_000), HlsTerminalCommitSubmissionDecision::Attempt { .. })); + let deadline = coordinator.owner_snapshot(&key).expect("initial owner").effective_deadline_ms; + let mut stale_media = retry_command("bounded-unavailable", 10_000, 2, 1); + stale_media.session = Arc::clone(&session); + assert!(matches!( + coordinator.submit(stale_media, 10_100), + HlsTerminalCommitSubmissionDecision::PendingExisting { .. } + )); + assert_eq!( + coordinator.owner_snapshot(&key).expect("unavailable owner survives stale media").effective_deadline_ms, + deadline + ); + let mut duplicate = retry_command("bounded-unavailable", 10_000, 3, 1); + duplicate.session = session; + duplicate.decision = + HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility::TerminalMediaNotReady); + assert!(matches!( + coordinator.submit(duplicate, deadline), + HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) + )); + assert_eq!(coordinator.owner_snapshot(&key).expect("bounded owner").effective_deadline_ms, deadline); + } } diff --git a/backend/hls/src/terminal_tail/plan.rs b/backend/hls/src/terminal_tail/plan.rs index 33ff5bd77..c92863d28 100644 --- a/backend/hls/src/terminal_tail/plan.rs +++ b/backend/hls/src/terminal_tail/plan.rs @@ -195,6 +195,7 @@ pub enum HlsLeasePlaybackMode { #[default] Live, TerminalTail(Arc), + TerminalCustomResponse(Arc), TerminalUnavailable { decision_generation: u64, reason: HlsTerminalTailCompatibility, diff --git a/backend/hls/src/terminal_tail/tests.rs b/backend/hls/src/terminal_tail/tests.rs index 2eafb74ae..5c07facb0 100644 --- a/backend/hls/src/terminal_tail/tests.rs +++ b/backend/hls/src/terminal_tail/tests.rs @@ -162,6 +162,7 @@ async fn unresolved_aes_terminal_base_evidence(iv: &str) -> HlsTerminalBaseEvide host_local_index: 0, }, proxy_seq, + media_start_ms: 0, duration_ms: 4_000, proxy_file_ext: "ts".into(), content_type: "video/mp2t".into(), @@ -238,6 +239,7 @@ async fn terminal_timing_session( host_local_index: 0, }, proxy_seq, + media_start_ms: proxy_seq * terminal_asset.duration_ms(), duration_ms: terminal_asset.duration_ms(), proxy_file_ext: "ts".into(), content_type: "video/mp2t".into(), diff --git a/backend/hls/src/timeline.rs b/backend/hls/src/timeline.rs index 08150830f..1d3072259 100644 --- a/backend/hls/src/timeline.rs +++ b/backend/hls/src/timeline.rs @@ -172,6 +172,8 @@ pub fn is_hls_provisioning_gap_segment(entry: &SegmentEntry) -> bool { pub struct SegmentEntry { pub origin_key: OriginSegmentKey, pub proxy_seq: u64, + /// Immutable session-local media offset; independent of the retained cache window. + pub media_start_ms: u64, pub duration_ms: u64, pub proxy_file_ext: String, pub content_type: String, @@ -193,6 +195,7 @@ impl fmt::Debug for SegmentEntry { f.debug_struct("SegmentEntry") .field("origin_key", &self.origin_key) .field("proxy_seq", &self.proxy_seq) + .field("media_start_ms", &self.media_start_ms) .field("duration_ms", &self.duration_ms) .field("proxy_file_ext", &self.proxy_file_ext) .field("content_type", &self.content_type) @@ -317,6 +320,7 @@ struct TimelineDraft { origin_epoch_sequence_base: Option, origin_seq_highwater: Option, proxy_next_seq: Option, + next_media_start_ms: u64, origin_to_proxy: HashMap, published_resource_history: HlsPublishedResourceHistory, discontinuity_sequence: u64, @@ -364,6 +368,7 @@ impl From<&HlsSession> for TimelineDraft { origin_epoch_sequence_base: session.origin_epoch_sequence_base, origin_seq_highwater: session.origin_seq_highwater, proxy_next_seq: session.proxy_next_seq, + next_media_start_ms: session.next_media_start_ms, origin_to_proxy: session.origin_to_proxy.clone(), published_resource_history: session.published_resource_history.clone(), discontinuity_sequence: session.discontinuity_sequence, @@ -482,6 +487,7 @@ impl HlsSession { self.origin_epoch_sequence_base = draft.origin_epoch_sequence_base; self.origin_seq_highwater = draft.origin_seq_highwater; self.proxy_next_seq = draft.proxy_next_seq; + self.next_media_start_ms = draft.next_media_start_ms; self.origin_to_proxy = draft.origin_to_proxy; self.published_resource_history = draft.published_resource_history; self.discontinuity_sequence = draft.discontinuity_sequence; @@ -502,7 +508,6 @@ impl HlsSession { /// /// The snapshot contains only metadata and is safe to evaluate after releasing the session lock. pub fn ready_timeline_snapshot(&self, first_proxy_seq: u64, now_ms: u64) -> HlsReadyTimelineSnapshot { - let mut start_ms = 0_u64; let mut expected_proxy_seq = first_proxy_seq; let mut units = Vec::new(); for (&proxy_seq, segment) in self.segments.range(first_proxy_seq..) { @@ -522,7 +527,7 @@ impl HlsSession { }); units.push(HlsReadyTimelineUnit { proxy_seq, - start_ms, + start_ms: segment.media_start_ms, duration_ms: segment.duration_ms, state: if matches!(segment.status, SegmentCacheStatus::Ready { .. }) { HlsReadyMediaState::Ready @@ -533,7 +538,6 @@ impl HlsSession { required_key_ready: segment.encryption.is_none() || key_ready_valid_until_ms.is_some(), key_ready_valid_until_ms, }); - start_ms = start_ms.saturating_add(segment.duration_ms); expected_proxy_seq = expected_proxy_seq.saturating_add(1); } HlsReadyTimelineSnapshot { units: Arc::from(units) } @@ -861,6 +865,13 @@ impl TimelineDraft { effective_host_id: u64, handoff_discontinuity_before: bool, ) -> Result { + let map_ref = parsed + .map_ref + .map(|map_id| { + let parsed_map = manifest.maps.get(map_id).ok_or(TimelineMapError::UnsupportedMapExtension)?; + self.map_origin_map(parsed_map) + }) + .transpose()?; let current_epoch_key = self.origin_segment_key(effective_host_id, parsed.origin_seq); if let Some(proxy_seq) = self.origin_to_proxy.get(¤t_epoch_key).copied() { if let Some(entry) = self.segments.get_mut(&proxy_seq) { @@ -871,6 +882,7 @@ impl TimelineDraft { valid_until_ms: None, }); entry.encryption = map_segment_encryption(parsed)?; + entry.map_ref = map_ref; } return Ok(proxy_seq); } @@ -883,18 +895,12 @@ impl TimelineDraft { let proxy_seq = self.proxy_next_seq.unwrap_or_default(); self.proxy_next_seq = Some(proxy_seq.checked_add(1).ok_or(TimelineMapError::ProxySequenceOverflow)?); - let map_ref = parsed - .map_ref - .map(|map_id| { - let parsed_map = manifest.maps.get(map_id).ok_or(TimelineMapError::UnsupportedMapExtension)?; - self.map_origin_map(parsed_map) - }) - .transpose()?; let proxy_file_ext = proxy_extension_from_url(&parsed.resolved_origin_url, SEGMENT_EXTENSIONS) .ok_or(TimelineMapError::UnsupportedSegmentExtension)?; let entry = SegmentEntry { origin_key, proxy_seq, + media_start_ms: self.next_media_start_ms, duration_ms: parsed.duration_ms, content_type: default_content_type_for_segment_ext(&proxy_file_ext).to_string(), cache_key: SegmentCacheKey::new(self.proxy_session_id.clone(), proxy_seq, &proxy_file_ext), @@ -916,6 +922,7 @@ impl TimelineDraft { access: Arc::new(CacheAccessState::new()), }; + self.next_media_start_ms = self.next_media_start_ms.saturating_add(parsed.duration_ms); self.origin_to_proxy.insert(origin_key, proxy_seq); self.segments.insert(proxy_seq, entry); Ok(proxy_seq) @@ -1901,4 +1908,36 @@ mod tests { assert_eq!(session.segments.get(&0).expect("first segment").map_ref, Some(ProxyMapId(0))); assert_eq!(session.segments.get(&1).expect("second segment").map_ref, Some(ProxyMapId(1))); } + + #[test] + fn media_clock_survives_complete_collection_and_origin_rollover_with_variable_durations() { + let mut session = session(); + session + .apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-MEDIA-SEQUENCE:100\n\ + #EXTINF:4,\n100.ts\n#EXTINF:6,\n101.ts\n#EXTINF:5,\n102.ts\n", + )) + .expect("initial variable timeline"); + assert_eq!( + session.segments.values().map(|entry| entry.media_start_ms).collect::>(), + vec![0, 4_000, 10_000] + ); + session.segments.clear(); + session.origin_to_proxy.clear(); + session + .apply_origin_rebase_manifest( + &normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-MEDIA-SEQUENCE:0\n\ + #EXTINF:6,\nnew-0.ts\n#EXTINF:3,\nnew-1.ts\n", + ), + 2, + false, + ) + .expect("rebased timeline after collection"); + assert_eq!( + session.segments.values().map(|entry| (entry.proxy_seq, entry.media_start_ms)).collect::>(), + vec![(3, 15_000), (4, 21_000)] + ); + assert_eq!(session.next_media_start_ms, 24_000); + } } diff --git a/backend/hls/src/transient.rs b/backend/hls/src/transient.rs index 860cd4b36..cfa397735 100644 --- a/backend/hls/src/transient.rs +++ b/backend/hls/src/transient.rs @@ -123,6 +123,7 @@ pub struct TransientResourceRef { pub expires_at_ms: u64, pub access: Arc, revision: TransientResourceRevision, + origin_account_generation: u64, } #[derive(Clone, Eq, PartialEq)] @@ -161,6 +162,8 @@ pub struct TransientObjectFetchToken { #[derive(Clone)] pub struct TransientPassthroughState { pub resources: HashMap, + invalid_origin_resources: HashSet, + origin_account_generation: u64, pub object_cache: HashMap, object_fetch_notifiers: HashMap>, next_resource_revision: u64, @@ -193,6 +196,8 @@ impl fmt::Debug for TransientPassthroughState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TransientPassthroughState") .field("resources_len", &self.resources.len()) + .field("invalid_origin_resources", &self.invalid_origin_resources.len()) + .field("origin_account_generation", &self.origin_account_generation) .field("object_cache_len", &self.object_cache.len()) .field("object_fetch_notifiers_len", &self.object_fetch_notifiers.len()) .field("next_resource_revision", &self.next_resource_revision) diff --git a/backend/hls/src/transient/identity.rs b/backend/hls/src/transient/identity.rs index 521426ce0..236fe3eb5 100644 --- a/backend/hls/src/transient/identity.rs +++ b/backend/hls/src/transient/identity.rs @@ -282,6 +282,15 @@ impl TransientResourceRef { expires_at_ms: now_ms.saturating_add(ttl_ms), access: Arc::new(CacheAccessState::new()), revision: TransientResourceRevision::default(), + origin_account_generation: 0, + } + } + + pub(crate) fn scope_to_origin_account(&mut self, generation: u64, rewrite_secret: &[u8]) { + self.origin_account_generation = generation; + if generation != 0 { + let identity = format!("account-generation:{generation}\0{}", self.resolved_origin_uri); + self.id = build_transient_resource_id(&identity, rewrite_secret); } } @@ -301,6 +310,7 @@ impl TransientResourceRef { pub(super) fn has_same_cache_identity(&self, other: &Self) -> bool { self.id == other.id + && self.origin_account_generation == other.origin_account_generation && self.kind == other.kind && self.encrypted_media == (self.encrypted_media || other.encrypted_media) && self.resolved_origin_uri == other.resolved_origin_uri @@ -324,6 +334,7 @@ impl fmt::Debug for TransientResourceRef { .field("expires_at_ms", &self.expires_at_ms) .field("active_readers", &self.access.active_readers()) .field("revision", &self.revision) + .field("origin_account_generation", &self.origin_account_generation) .finish() } } diff --git a/backend/hls/src/transient/manifests.rs b/backend/hls/src/transient/manifests.rs index dd82b9495..0f45b2d7b 100644 --- a/backend/hls/src/transient/manifests.rs +++ b/backend/hls/src/transient/manifests.rs @@ -28,6 +28,8 @@ impl TransientPassthroughState { pub fn new(resource_ttl_ms: u64) -> Self { Self { resources: HashMap::new(), + invalid_origin_resources: HashSet::new(), + origin_account_generation: 0, object_cache: HashMap::new(), object_fetch_notifiers: HashMap::new(), next_resource_revision: 1, diff --git a/backend/hls/src/transient/resources.rs b/backend/hls/src/transient/resources.rs index f40303d9c..8dcc72864 100644 --- a/backend/hls/src/transient/resources.rs +++ b/backend/hls/src/transient/resources.rs @@ -6,11 +6,29 @@ use super::{ use std::{collections::HashSet, sync::Arc}; impl TransientPassthroughState { + pub(crate) fn invalidate_origin_resources(&mut self) { + self.invalid_origin_resources.extend(self.resources.keys().cloned()); + self.origin_account_generation = self.origin_account_generation.saturating_add(1); + self.rolling_event_rewrite_seed = None; + } + + pub(crate) fn origin_account_generation(&self) -> u64 { self.origin_account_generation } + + /// Whether this mapping may initiate origin work for the current account. + pub fn origin_resource_is_fetchable(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { + resource.origin_account_generation == self.origin_account_generation + && !self.invalid_origin_resources.contains(&resource.id) + && self.resource_matches_current(resource, now_ms) + } + pub fn upsert_resources(&mut self, resources: I) where I: IntoIterator, { for mut resource in resources { + if resource.origin_account_generation == self.origin_account_generation { + self.invalid_origin_resources.remove(&resource.id); + } let preserve_revision = self .resources .get(&resource.id) @@ -123,6 +141,7 @@ impl TransientPassthroughState { .map(|(id, _)| id.clone()) .collect::>(); self.resources.retain(|id, _| !expired_resource_ids.contains(id)); + self.invalid_origin_resources.retain(|id| !expired_resource_ids.contains(id)); } pub fn active_resource_readers(&self) -> u32 { diff --git a/backend/hls/src/transient/tests/streaming.rs b/backend/hls/src/transient/tests/streaming.rs index d8ff5c2bb..060778768 100644 --- a/backend/hls/src/transient/tests/streaming.rs +++ b/backend/hls/src/transient/tests/streaming.rs @@ -67,3 +67,84 @@ fn expired_finalized_object_is_refetched_instead_of_served_stale() { super::super::TransientObjectFetchDecision::Fetch(_) )); } + +#[test] +fn account_handoff_retains_cached_transient_bytes_but_blocks_old_origin_mapping() { + let mut state = TransientPassthroughState::default(); + let proxy = ProxySessionId("account-handoff".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/old/media.ts", + b"secret", + 1, + 100, + Some("ts".to_string()), + ); + let id = resource.id.clone(); + state.upsert_resources([resource]); + let resource = state.resolve_current_resource(&id, 2).unwrap(); + let token = match state.begin_object_fetch(&proxy, &resource, "ts", 2, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("uncached media starts one fetch") + } + }; + assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 3, 100)); + state.invalidate_origin_resources(); + let key = TransientPassthroughState::transient_object_key(&proxy, &id, "ts"); + assert!(state.ready_object(&key, TransientResourceKind::Segment, 4).is_some()); + assert!(state.resolve_current_resource(&id, 4).is_some()); + assert!(!state.origin_resource_is_fetchable(&resource, 4)); + state.upsert_resources([resource.clone()]); + assert!(!state.origin_resource_is_fetchable(&resource, 5)); + assert!(state.ready_object(&key, TransientResourceKind::Segment, 5).is_some()); + state.invalidate_origin_resources(); + state.prune_expired(102); + assert!(state.invalid_origin_resources.is_empty()); +} + +#[test] +fn account_handoff_gives_identical_key_uri_a_new_cache_identity_and_retains_the_old_key() { + let mut state = TransientPassthroughState::default(); + let proxy = ProxySessionId("key-handoff".to_string()); + let body = "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"shared.key\"\n#EXTINF:4,\nmedia.ts\n"; + let old = TransientManifestRewriter::rewrite(body, "https://origin.example/index.m3u8", &proxy, b"secret", 1, 100); + let old_key = old.resources.iter().find(|resource| resource.kind == TransientResourceKind::Key).unwrap().id.clone(); + state.upsert_resources(old.resources); + let old_key_ref = state.resolve_current_resource(&old_key, 2).unwrap(); + let super::super::TransientObjectFetchDecision::Fetch(token) = + state.begin_object_fetch(&proxy, &old_key_ref, "key", 2, 100) + else { + panic!("old key starts its first cache fill") + }; + assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 3, 100)); + state.invalidate_origin_resources(); + let new = TransientManifestRewriter::rewrite_with_options( + body, + "https://origin.example/index.m3u8", + &proxy, + b"secret", + 4, + 100, + crate::transient_manifest::TransientRewriteOptions { + origin_account_generation: state.origin_account_generation(), + ..crate::transient_manifest::TransientRewriteOptions::default() + }, + ); + let new_key = new.resources.iter().find(|resource| resource.kind == TransientResourceKind::Key).unwrap().id.clone(); + assert_ne!(old_key, new_key); + assert_ne!(old.body, new.body); + state.upsert_resources(new.resources); + let new_key_ref = state.resolve_current_resource(&new_key, 5).unwrap(); + assert_eq!(old_key_ref.resolved_origin_uri, new_key_ref.resolved_origin_uri); + let old_lookup = TransientPassthroughState::transient_object_key(&proxy, &old_key, "key"); + let new_lookup = TransientPassthroughState::transient_object_key(&proxy, &new_key, "key"); + assert!(state.ready_object(&old_lookup, TransientResourceKind::Key, 5).is_some()); + assert!(state.ready_object(&new_lookup, TransientResourceKind::Key, 5).is_none()); + assert!(!state.origin_resource_is_fetchable(&old_key_ref, 5)); + assert!(state.origin_resource_is_fetchable(&new_key_ref, 5)); + assert!(matches!( + state.begin_object_fetch(&proxy, &new_key_ref, "key", 5, 100), + super::super::TransientObjectFetchDecision::Fetch(_) + )); +} diff --git a/backend/hls/src/transient_fetcher/tests/lifecycle.rs b/backend/hls/src/transient_fetcher/tests/lifecycle.rs index e628a5335..eadc907bd 100644 --- a/backend/hls/src/transient_fetcher/tests/lifecycle.rs +++ b/backend/hls/src/transient_fetcher/tests/lifecycle.rs @@ -149,7 +149,7 @@ async fn dropping_direct_segment_body_is_client_abort_without_state_transition() assert_eq!(fixture.segment_failure_count().await, 1); assert_eq!(fixture.resource.access.active_readers(), 0); assert_eq!(origin.requests.lock().await.len(), 1); - assert!(take_body_failure_log_attempts().is_empty()); + assert_eq!(take_body_failure_log_attempts().len(), 0); } #[tokio::test] @@ -285,6 +285,9 @@ async fn transient_origin_io_guard_finish_clean_decrements_work_once() { reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: Some(started_gen), + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let guard = HlsTransientOriginIoGuard::new( @@ -328,6 +331,9 @@ async fn transient_origin_io_guard_drop_under_lock_contention_decrements_work() reservation_ttl_secs: 60, preacquired_provider_handle: None, started_generation: Some(started_gen), + prepared_account: None, + playback_kind: tuliprox_core::model::PlaybackKind::LiveHls, + account_runtime: None, }; let guard = HlsTransientOriginIoGuard::new( diff --git a/backend/hls/src/transient_manifest.rs b/backend/hls/src/transient_manifest.rs index b92490087..516b94675 100644 --- a/backend/hls/src/transient_manifest.rs +++ b/backend/hls/src/transient_manifest.rs @@ -72,11 +72,14 @@ pub struct TransientRewriteResult { #[derive(Debug, Clone, Default)] pub struct TransientRewriteCheckpoint { encryption_state: TransientEncryptionState, + origin_account_generation: u64, } #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] pub struct TransientRewriteOptions { pub handoff_discontinuity_sequence: Option, + /// Separates account-dependent resources without changing their concrete origin URL. + pub origin_account_generation: u64, } /// Rewrites only HLS URI surfaces to transient live-HLS proxy resources. @@ -88,6 +91,7 @@ impl TransientManifestRewriter { /// `final_manifest_url` must be the concrete URL of the manifest that was actually fetched. For provider-url /// failover and HTTP redirects this can be the selected mirror or final CDN/origin host. Relative segment, MAP and /// key URIs are resolved against this URL so transient resource fetches keep working after redirects. + #[cfg(test)] pub fn rewrite( body: &str, final_manifest_url: &str, @@ -127,7 +131,10 @@ impl TransientManifestRewriter { reverse_proxy_rewrite_secret, now_ms, ttl_ms, - TransientRewriteCheckpoint::default(), + TransientRewriteCheckpoint { + origin_account_generation: options.origin_account_generation, + ..TransientRewriteCheckpoint::default() + }, ); if let Some(discontinuity_sequence) = options.handoff_discontinuity_sequence { result.body = apply_handoff_discontinuity_boundary(&result.body, discontinuity_sequence); @@ -178,6 +185,7 @@ fn rewrite_transient_fragment( now_ms, ttl_ms, resources: &mut resources, + origin_account_generation: checkpoint.origin_account_generation, }; for part in body.split_inclusive('\n') { processed_line_count = processed_line_count.saturating_add(1); @@ -340,6 +348,7 @@ struct TransientResourceRewriteContext<'a> { now_ms: u64, ttl_ms: u64, resources: &'a mut HashMap, + origin_account_generation: u64, } impl TransientResourceRewriteContext<'_> { @@ -396,6 +405,7 @@ impl TransientResourceRewriteContext<'_> { self.ttl_ms, Some(extension.clone()), ); + resource.scope_to_origin_account(self.origin_account_generation, self.reverse_proxy_rewrite_secret); resource.encrypted_media = encrypted_media; let resource_id = resource.id.clone(); self.resources @@ -780,7 +790,7 @@ mod tests { b"secret", 100, 1_000, - TransientRewriteOptions { handoff_discontinuity_sequence: Some(0) }, + TransientRewriteOptions { handoff_discontinuity_sequence: Some(0), ..TransientRewriteOptions::default() }, ) } diff --git a/backend/iptv/src/stalker/cookie_jar.rs b/backend/iptv/src/stalker/cookie_jar.rs index aad820778..06dec6408 100644 --- a/backend/iptv/src/stalker/cookie_jar.rs +++ b/backend/iptv/src/stalker/cookie_jar.rs @@ -250,6 +250,6 @@ mod tests { let jar = StalkerCookieJar::new(); jar.ingest_set_cookie(["stb_lang=en; Max-Age=10"], 1_000); assert!(jar.active_cookie_header(1_009).contains("stb_lang=en")); - assert!(jar.active_cookie_header(1_010).is_empty()); + assert_eq!(jar.active_cookie_header(1_010).len(), 0); } } diff --git a/backend/iptv/src/xtream/mod.rs b/backend/iptv/src/xtream/mod.rs index 1a1d35857..c4447c929 100644 --- a/backend/iptv/src/xtream/mod.rs +++ b/backend/iptv/src/xtream/mod.rs @@ -1038,14 +1038,14 @@ mod tests { let sink = RecordingSink::default(); // One second before the window opens. super::notify_account_expire_at(EXPIRY - super::THREE_DAYS_IN_SECS - 1, Some(EXPIRY), &sink, "u", "p"); - assert!(sink.states().is_empty()); + assert_eq!(sink.states().len(), 0); } #[test] fn no_expiry_date_emits_nothing() { let sink = RecordingSink::default(); super::notify_account_expire_at(EXPIRY, None, &sink, "user", "provider"); - assert!(sink.states().is_empty()); + assert_eq!(sink.states().len(), 0); } fn options_with_flags(flags: &[ConfigInputFlags]) -> ConfigInputOptions { @@ -1173,7 +1173,7 @@ mod tests { assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); assert_eq!(direct_fetch.groups.len(), 1); assert_eq!(direct_fetch.groups[0].channels.len(), 89); - assert!(direct_fetch.quality_rejections.is_empty()); + assert_eq!(direct_fetch.quality_rejections.len(), 0); let mut update_fetch = PlaylistFetch::default(); apply_in_memory_cluster_download( @@ -1241,7 +1241,7 @@ mod tests { .expect("deduplicated candidate should be accepted"); assert_eq!(baseline_reads.load(Ordering::SeqCst), 1); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); assert_eq!( fetch.quality_acceptances, vec![ClusterUpdateAcceptance { @@ -1319,8 +1319,8 @@ mod tests { assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); assert_eq!(direct_fetch.groups.iter().map(|group| group.channels.len()).sum::(), 2); assert_eq!(disabled_fetch.groups.iter().map(|group| group.channels.len()).sum::(), 2); - assert!(direct_fetch.quality_rejections.is_empty()); - assert!(disabled_fetch.quality_rejections.is_empty()); + assert_eq!(direct_fetch.quality_rejections.len(), 0); + assert_eq!(disabled_fetch.quality_rejections.len(), 0); } #[tokio::test] @@ -1359,7 +1359,7 @@ mod tests { .expect("forced candidate should be accepted"); assert_eq!(baseline_reads.load(Ordering::SeqCst), 0); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); assert_eq!(fetch.groups.len(), 1); assert_eq!(fetch.groups[0].channels.len(), 2); assert_eq!(fetch.groups[0].channels[0].header.name.as_ref(), "winning-duplicate"); @@ -1409,7 +1409,7 @@ mod tests { vec![candidate_group(XtreamCluster::Live, 1, 1)], ); assert_eq!(disabled.groups[0].channels.len(), 1); - assert!(disabled.quality_rejections.is_empty()); + assert_eq!(disabled.quality_rejections.len(), 0); let mut bootstrap = PlaylistFetch::default(); apply_cluster_update_quality( @@ -1420,7 +1420,7 @@ mod tests { vec![candidate_group(XtreamCluster::Video, 2, 3)], ); assert_eq!(bootstrap.groups[0].channels.len(), 3); - assert!(bootstrap.quality_rejections.is_empty()); + assert_eq!(bootstrap.quality_rejections.len(), 0); let mut empty_bootstrap = PlaylistFetch::default(); apply_cluster_update_quality(&mut empty_bootstrap, XtreamCluster::Series, None, 90, Vec::new()); diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs index 1df0ebae8..9415954f3 100644 --- a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs @@ -12,7 +12,7 @@ fn pipeline_transparency_telemetry_reports_non_cluster_cache_without_synthetic_c assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); assert_eq!(telemetry.source, Some(PlaylistUpdateDataSource::Cache)); - assert!(telemetry.clusters.is_empty()); + assert_eq!(telemetry.clusters.len(), 0); } #[test] diff --git a/backend/processing/src/processor/playlist/tests/curation_publication.rs b/backend/processing/src/processor/playlist/tests/curation_publication.rs index ad84eac27..84d9b3a54 100644 --- a/backend/processing/src/processor/playlist/tests/curation_publication.rs +++ b/backend/processing/src/processor/playlist/tests/curation_publication.rs @@ -342,7 +342,7 @@ impl Publication { .all(|item| text.contains(item.header.title.as_ref()) && text.contains(item.header.url.as_ref())) })); let strm = self.strm_contents(); - assert!(!strm.is_empty()); + assert_ne!(strm.len(), 0); let files = file_snapshot(self.directory.path()); let live_title = expected[0].channels[0].header.title.as_bytes(); assert!( @@ -352,7 +352,7 @@ impl Publication { }), "new EPG programme must be published" ); - assert!(!self.cache_signature().await.is_empty()); + assert_ne!(self.cache_signature().await.len(), 0); } async fn identity_signature(&self) -> BTreeMap { @@ -455,7 +455,7 @@ async fn tmdb_https_selection_only_publishes_m3u_strm_without_an_xtream_output() let text = std::fs::read_to_string(run.directory.path().join("published.m3u")).unwrap(); assert!(text.contains("First") && text.contains("Show S01E01") && text.contains("Live")); assert!(!text.contains("Unselected") && !text.contains("TMDB")); - assert!(!run.strm_contents().is_empty()); + assert_ne!(run.strm_contents().len(), 0); let cache = run.context.playlist_state.as_ref().unwrap().data.read().await; assert!(cache[&run.target.name].xtream.is_none()); assert_eq!(server.requests.lock().unwrap().len(), 3); @@ -469,7 +469,7 @@ async fn tmdb_https_empty_and_no_match_clear_published_vod_series_but_keep_live( let server = DiscoveryServer::start().await; let run = Publication::new(&server, &target("curated", false, true)); run.publish().await.unwrap(); - assert!(!run.strm_contents().is_empty()); + assert_ne!(run.strm_contents().len(), 0); server.reply(MOVIES, 200, body); server.reply(TV, 200, body); run.publish().await.unwrap(); diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs index bf9907230..c9dae7600 100644 --- a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs +++ b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs @@ -248,7 +248,7 @@ async fn curation_production_wiring_does_not_fall_back_to_the_generic_client() { server.requests.lock().unwrap().is_empty(), "generic client has the fixture CA and could succeed, but is not used" ); - assert!(!run.cache_signature().await.is_empty()); + assert_ne!(run.cache_signature().await.len(), 0); assert!(!file_snapshot(run.directory.path()).is_empty()); } @@ -272,7 +272,7 @@ async fn curation_failure_respects_empty_input_authorization() { assert_eq!(result.is_err(), authorized, "ordinary M3U persistence still rejects an empty playlist"); assert_eq!(server.requests.lock().unwrap().len(), if authorized { 3 } else { 0 }); assert!(file_snapshot(run.directory.path()).is_empty()); - assert!(run.cache_signature().await.is_empty()); + assert_eq!(run.cache_signature().await.len(), 0); } } diff --git a/backend/processing/src/processor/playlist/tests/mapping_stage.rs b/backend/processing/src/processor/playlist/tests/mapping_stage.rs index 95aa81519..598ad1fa3 100644 --- a/backend/processing/src/processor/playlist/tests/mapping_stage.rs +++ b/backend/processing/src/processor/playlist/tests/mapping_stage.rs @@ -153,7 +153,7 @@ async fn m3u_alias_playlist_is_downloaded_and_indexed_separately() { assert!(errors.is_empty(), "unexpected download errors: {errors:?}"); assert!(storage_error.is_none(), "unexpected primary storage error: {storage_error:?}"); assert!(!partial); - assert!(quality_rejections.is_empty()); + assert_eq!(quality_rejections.len(), 0); assert!(!primary_playlist.is_empty()); let alias_url = tuliprox_repository::load_input_m3u_stream_url( &ctx.config, @@ -214,7 +214,7 @@ async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { assert!(!first_errors.is_empty(), "missing alias should report an error"); assert!(first_storage_error.is_none(), "primary storage should succeed: {first_storage_error:?}"); assert!(!first_partial); - assert!(first_quality_rejections.is_empty()); + assert_eq!(first_quality_rejections.len(), 0); assert!(!first_playlist.is_empty()); assert!(ctx.is_input_downloaded("main-account").await); assert!(!ctx.is_input_downloaded("retry-account").await); @@ -238,7 +238,7 @@ async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { assert!(second_errors.is_empty(), "unexpected retry errors: {second_errors:?}"); assert!(second_storage_error.is_none(), "primary storage should remain readable: {second_storage_error:?}"); assert!(!second_partial); - assert!(second_quality_rejections.is_empty()); + assert_eq!(second_quality_rejections.len(), 0); assert!(!second_playlist.is_empty()); assert!(ctx.is_input_downloaded("retry-account").await); let alias_url = tuliprox_repository::load_input_m3u_stream_url( diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs index 16c8d9c3a..c20148d07 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs @@ -315,8 +315,8 @@ async fn playlist_update_run_manual_and_automatic_execution_order_is_generated_a }; let manual_progress = progress_for(&manual_run_id); let automatic_progress = progress_for(&automatic_run_id); - assert!(!manual_progress.is_empty()); - assert!(!automatic_progress.is_empty()); + assert_ne!(manual_progress.len(), 0); + assert_ne!(automatic_progress.len(), 0); assert!(manual_progress.iter().all(|progress| progress.execution_order == Some(manual_order))); assert!(automatic_progress.iter().all(|progress| progress.execution_order == Some(automatic_order))); } @@ -689,11 +689,11 @@ async fn coderabbit_stalker_global_failures_mark_only_requested_non_skipped_clus assert_eq!(fetch.errors.len(), 1, "{failure:?}"); assert!(!fetch.is_ok()); assert!(fetch.quality_acceptances.is_empty() && fetch.quality_rejections.is_empty()); - assert!(fetch.force_updates.is_empty()); + assert_eq!(fetch.force_updates.len(), 0); let requests = server.finish(); match failure { - FailurePoint::Handshake => assert!(!requests.is_empty()), - FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert!(requests.is_empty()), + FailurePoint::Handshake => assert_ne!(requests.len(), 0), + FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert_eq!(requests.len(), 0), } } } @@ -727,7 +727,7 @@ async fn coderabbit_xtream_http_failure_preserves_successful_disk_clusters() { assert_eq!(fetch.failed_clusters, vec![XtreamCluster::Video]); assert!(!fetch.is_ok(), "the failed cluster must remain a technical failure"); assert!(fetch.persisted); - assert!(fetch.quality_rejections.is_empty()); + assert_eq!(fetch.quality_rejections.len(), 0); let accepted: Vec<_> = match quality { UpdateQualityPolicy::Enforce => fetch.quality_acceptances.iter().map(|value| value.cluster).collect(), UpdateQualityPolicy::Bypass => fetch.force_updates.iter().map(|value| value.cluster).collect(), @@ -807,7 +807,7 @@ async fn pipeline_transparency_normal_policy_reuses_a_valid_cluster_cache() { assert!(result.errors.is_empty()); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.job_state(), InputJobState::Ready); assert_baseline(&result.source.take_groups()); assert!(requests.is_empty(), "valid cache should avoid provider requests: {requests:?}"); @@ -1217,7 +1217,7 @@ async fn force_policy_retains_a_cluster_after_a_technical_parse_failure() { assert!(!result.errors.is_empty(), "malformed VOD response must remain a technical error"); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD"); assert_eq!(vod.title.as_ref(), "old-vod"); @@ -1280,7 +1280,7 @@ async fn in_memory_series_quality_compares_catalog_rows_with_an_enriched_baselin let requests = server.finish(); assert!(result.errors.is_empty(), "Series update failed: {:?}", result.errors); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); assert_eq!(groups.len(), 1); assert_eq!(groups[0].title.as_ref(), "candidate-series"); @@ -1390,7 +1390,7 @@ async fn in_memory_quality_accepts_unique_population_and_persists_the_last_dupli assert!(result.errors.is_empty(), "deduplicated acceptance failed: {:?}", result.errors); assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let groups = result.source.take_groups(); assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); let winner_group = groups.iter().find(|group| group.id == 2).expect("winning category"); @@ -1445,7 +1445,7 @@ async fn m3u_update_quality_enforces_the_existing_cluster_threshold() { let mut initial = download_input(&initial_ctx, &input, false).await; assert!(initial.errors.is_empty(), "initial M3U download failed: {:?}", initial.errors); - assert!(initial.quality_rejections.is_empty()); + assert_eq!(initial.quality_rejections.len(), 0); assert_eq!(initial.source.get_channel_count(), 2); tokio::fs::write(&playlist_path, "#EXTM3U\n#EXTINF:-1,New\nhttp://new.example/1\n") diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs index 7e93b080d..eaca5ceef 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs @@ -105,7 +105,7 @@ async fn m3u_update_quality_all_clusters_accepted() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; let mut result = run(&ctx, &input, [9, 11, 10]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.update_state(), PlaylistUpdateState::Success); verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; } @@ -257,7 +257,7 @@ async fn m3u_update_quality_disabled() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await; let mut result = run(&ctx, &input, [1, 1, 0]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert!(result .input_telemetry .as_ref() @@ -273,7 +273,7 @@ async fn m3u_update_quality_first_import() { let temp = tempfile::tempdir().unwrap(); let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await; let mut result = run(&ctx, &input, [1, 1, 1]).await; - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none() && c.baseline_count.is_none() && c.decision == Some(PlaylistUpdateClusterDecision::Accepted))); @@ -292,7 +292,7 @@ async fn m3u_update_quality_download_or_parse_failure() { } let mut result = download_input(&ctx, &input, false).await; assert!(!result.errors.is_empty()); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); assert_eq!(result.update_state(), PlaylistUpdateState::Failure); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); @@ -324,7 +324,7 @@ async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries); let mut result = run(&ctx, &input, [10, 10, 0]).await; assert_eq!(result.update_state(), PlaylistUpdateState::Success); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]); assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); @@ -388,7 +388,7 @@ async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() { tokio::fs::write(&input.url, body).await.unwrap(); let mut result = download_input(&ctx, &input, false).await; assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(result.quality_rejections.is_empty()); + assert_eq!(result.quality_rejections.len(), 0); let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); assert_eq!(counts(&saved.take_groups()), [10; 3]); } diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs index 0d2bd45fe..39db5fba2 100644 --- a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs @@ -93,7 +93,7 @@ async fn m3u_update_quality_playlist_update_status_staged_completion_is_independ let telemetry = completions[0].input_telemetry.as_ref().unwrap(); assert_eq!(telemetry.source, None); assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); - assert!(staged_completion::input_completions(&events, parent.id).is_empty()); + assert_eq!(staged_completion::input_completions(&events, parent.id).len(), 0); } } @@ -243,7 +243,7 @@ async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_ let skipped = download_input(&ctx, &staged, false).await; assert!(skipped.input_telemetry.is_none()); assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none()); - assert!(staged_completion::input_completions(&events, staged.id).is_empty()); + assert_eq!(staged_completion::input_completions(&events, staged.id).len(), 0); let result = download_input(&ctx, &parent, false).await; let status = input_cache::load_input_status(&staged_path); diff --git a/backend/processing/src/processor/stalker.rs b/backend/processing/src/processor/stalker.rs index e4edc956a..f5fe926a2 100644 --- a/backend/processing/src/processor/stalker.rs +++ b/backend/processing/src/processor/stalker.rs @@ -667,7 +667,7 @@ mod tests { fn playback_candidates_are_empty_without_any_command() { let item = stalker_item("", None); - assert!(playback_candidates(&item).is_empty()); + assert_eq!(playback_candidates(&item).len(), 0); } #[test] diff --git a/backend/processing/src/processor/stalker_refresh.rs b/backend/processing/src/processor/stalker_refresh.rs index a8c2a4b08..5ab39e278 100644 --- a/backend/processing/src/processor/stalker_refresh.rs +++ b/backend/processing/src/processor/stalker_refresh.rs @@ -1316,7 +1316,7 @@ mod tests { finish_completed_refresh(&app_config, temp.path(), identity, &checkpoint, update_quality(0, 0, 100)) .await?; - assert!(rejections.is_empty()); + assert_eq!(rejections.len(), 0); let published = load_active_manifest(temp.path(), identity).await?; assert_eq!(published.series.as_ref().map(|files| files.generation), Some(generation)); Ok(()) @@ -1379,7 +1379,7 @@ mod tests { ) .await?; - assert!(reports.quality_rejections.is_empty()); + assert_eq!(reports.quality_rejections.len(), 0); assert_eq!( reports.force_updates, vec![ClusterForceUpdate { cluster: XtreamCluster::Video, candidate_count: 0, configured_threshold: 100 }] @@ -1423,7 +1423,7 @@ mod tests { ) .await?; - assert!(reports.quality_rejections.is_empty()); + assert_eq!(reports.quality_rejections.len(), 0); assert_eq!(reports.force_updates.len(), 1); assert_eq!(reports.force_updates[0].cluster, XtreamCluster::Live); let published = load_active_manifest(temp.path(), identity).await?; diff --git a/backend/repository/src/alias_repository.rs b/backend/repository/src/alias_repository.rs index 091c75f5d..57a3b0fb3 100644 --- a/backend/repository/src/alias_repository.rs +++ b/backend/repository/src/alias_repository.rs @@ -1302,7 +1302,7 @@ url;other;other;http://third/get.php?username=user&password=pass;0;2;\n"; ) .await?; assert!(!receipt.changed); - assert!(receipt.matched.is_empty()); + assert_eq!(receipt.matched.len(), 0); assert_eq!(tokio::fs::read_to_string(&path).await?, original); Ok(()) } @@ -1397,7 +1397,7 @@ input_1;user;password;http://provider.tv;1;0;keep-me;old-enabled;old-expiry;trai .await?; assert!(!changed); - assert!(updated.is_empty()); + assert_eq!(updated.len(), 0); assert!(!backup_dir.exists()); Ok(()) } diff --git a/backend/repository/src/lineup_guide_index.rs b/backend/repository/src/lineup_guide_index.rs index 539f1634d..4f39477dd 100644 --- a/backend/repository/src/lineup_guide_index.rs +++ b/backend/repository/src/lineup_guide_index.rs @@ -485,7 +485,7 @@ mod tests { let playlist = vec![group(1, XtreamCluster::Series, vec![series])]; // `m3u_write_playlist` does not store series info, so the index must not point at it. - assert!(entries(&build_lineup_guide_index(&playlist, LineupGuideView::M3u)).is_empty()); + assert_eq!(entries(&build_lineup_guide_index(&playlist, LineupGuideView::M3u)).len(), 0); } #[test] @@ -529,7 +529,7 @@ mod tests { // `lineup.json` publishes the empty EPG id as the guide number of the first channel, which // no request can ask for, so neither the empty id nor the name may answer for it. - assert!(entries(&tree).is_empty()); + assert_eq!(entries(&tree).len(), 0); } #[tokio::test] diff --git a/backend/repository/src/playlist_repository/tests/lifecycle.rs b/backend/repository/src/playlist_repository/tests/lifecycle.rs index 355e9b3f4..24df6fb09 100644 --- a/backend/repository/src/playlist_repository/tests/lifecycle.rs +++ b/backend/repository/src/playlist_repository/tests/lifecycle.rs @@ -54,5 +54,5 @@ async fn intentional_empty_strm_cleanup_false_removes_indexed_files_but_keeps_un get_target_storage_path(&config, &target.name).expect("target storage") }; let index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm-retained")), &target_storage); - assert!(std::fs::read_to_string(index).expect("empty STRM index").is_empty()); + assert_eq!(std::fs::read_to_string(index).expect("empty STRM index").len(), 0); } diff --git a/backend/repository/src/playlist_repository/tests/transport.rs b/backend/repository/src/playlist_repository/tests/transport.rs index 7b4837dd1..90f3bc745 100644 --- a/backend/repository/src/playlist_repository/tests/transport.rs +++ b/backend/repository/src/playlist_repository/tests/transport.rs @@ -102,7 +102,7 @@ async fn intentional_empty_output_views_clear_managed_artifacts_without_alias_le "[]" ); let strm_index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm")), &target_storage); - assert!(std::fs::read_to_string(strm_index).expect("empty STRM index").is_empty()); + assert_eq!(std::fs::read_to_string(strm_index).expect("empty STRM index").len(), 0); { let cache = playlist_state.data.read().await; let cached = cache.get(&target.name).expect("empty target cache"); diff --git a/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs index b320a265a..b5ae4fed0 100644 --- a/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs +++ b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs @@ -190,8 +190,8 @@ async fn disk_force_keeps_active_files_after_a_technical_staging_failure() { .await; assert_eq!(result.errors.len(), 1); - assert!(result.outcomes.is_empty()); - assert!(result.force_updates.is_empty()); + assert_eq!(result.outcomes.len(), 0); + assert_eq!(result.force_updates.len(), 0); assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); assert_no_refresh_artifacts(&storage_path); } @@ -233,7 +233,7 @@ async fn pipeline_transparency_disk_quality_survives_post_evaluation_staging_fai quality: Some(95), }] ); - assert!(result.outcomes.is_empty()); + assert_eq!(result.outcomes.len(), 0); assert_eq!(result.errors.len(), 1); assert_eq!(result.failed_cluster, Some(XtreamCluster::Live)); assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); diff --git a/backend/session/src/active_user_manager/connections.rs b/backend/session/src/active_user_manager/connections.rs index a44a5a026..09627c6c0 100644 --- a/backend/session/src/active_user_manager/connections.rs +++ b/backend/session/src/active_user_manager/connections.rs @@ -12,10 +12,7 @@ use arc_swap::ArcSwapOption; use lru::LruCache; use shared::{ defaults::{default_grace_period_millis, default_grace_period_timeout_secs}, - model::{ - ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamChannel, StreamInfo, - UserConnectionPermission, VirtualId, - }, + model::{CustomVideoStreamType, StreamChannel, StreamInfo, UserConnectionPermission, VirtualId}, utils::{current_time_secs, sanitize_sensitive_info, strip_port, Internable}, }; use std::{ @@ -690,8 +687,7 @@ impl ActiveUserManager { } if let Some(stream_info) = preserved_update.as_ref() { - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + self.emit_current_stream_update(stream_info).await; } if connection_changed { @@ -870,7 +866,7 @@ impl ActiveUserManager { } for stream_info in preserved_updates { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.emit_current_stream_update(&stream_info).await; } for username in &disconnected_users { diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index d91d17960..64e2ddec1 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -173,7 +173,7 @@ pub enum StreamRequestDetach { } /// How long an ended session token blocks its recreation from a sealed playlist token. -const ENDED_SESSION_RECREATE_BLOCK: Duration = Duration::from_secs(600); +const ENDED_SESSION_RECREATE_BLOCK: Duration = Duration::from_mins(10); pub struct ReleasedConnection { pub addr_removed: bool, @@ -265,7 +265,6 @@ mod session_lifecycle; use self::observability::divergence_key; #[cfg(test)] use self::observability::DivergenceKind; -pub use self::session_lifecycle::next_session_incarnation; use self::{ connections::{ create_socket_reentry_guard_key, decide_connection_kind, url_host_key, PromotionAction, RecentWinnerProtection, @@ -279,3 +278,4 @@ use self::{ uses_session_reentry_guard, UserSessionParams, }, }; +pub use self::{observability::SharedStreamProviderProjection, session_lifecycle::next_session_incarnation}; diff --git a/backend/session/src/active_user_manager/observability.rs b/backend/session/src/active_user_manager/observability.rs index ad6755b91..058435252 100644 --- a/backend/session/src/active_user_manager/observability.rs +++ b/backend/session/src/active_user_manager/observability.rs @@ -13,6 +13,34 @@ use std::{ }; use tuliprox_core::utils::debug_if_enabled; +/// Locks viewer rows before an origin publication, so publication and its UI events have no cancellation gap. +pub struct SharedStreamProviderProjection<'a> { + connections: super::UserConnectionsWriteGuard<'a>, + events: &'a EventManager, + shared_stream_id: u64, +} + +impl SharedStreamProviderProjection<'_> { + /// Changes only display metadata and publishes each full changed row while updates are ordered by the lock. + pub fn update_provider(&mut self, provider: &Arc) -> usize { + let mut changed = 0; + for data in self.connections.by_key.values_mut() { + for stream in &mut data.streams { + if stream.channel.shared + && stream.channel.shared_stream_id == Some(self.shared_stream_id) + && stream.provider != *provider + { + stream.provider.clone_from(provider); + self.events + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream.clone()))); + changed += 1; + } + } + } + changed + } +} + pub(super) struct DivergenceEntry { pub(super) last_logged: Instant, pub(super) count_since_last_log: u32, @@ -44,6 +72,18 @@ pub(super) struct DivergenceSnapshot { } impl ActiveUserManager { + /// Acquire before the HLS session write lock; do not call other manager methods while this guard is held. + pub async fn lock_shared_stream_provider_projection( + &self, + shared_stream_id: u64, + ) -> SharedStreamProviderProjection<'_> { + SharedStreamProviderProjection { + connections: self.write_connections().await, + events: &self.event_manager, + shared_stream_id, + } + } + pub(super) fn custom_stream_technical_info() -> StreamTechnicalInfo { StreamTechnicalInfo { container: String::from("mpegts"), @@ -143,6 +183,23 @@ impl ActiveUserManager { } } + /// Re-read an existing playback row and keep the connection lock through its synchronous event send. + /// A delayed producer must neither restore an older projection nor recreate a removed playback. + pub(crate) async fn emit_current_stream_update(&self, snapshot: &StreamInfo) { + let connections = self.connections.read().await; + let current = connections.by_key.get(&snapshot.username).and_then(|data| { + data.streams.iter().find(|stream| { + stream.uid == snapshot.uid + && stream.session_token == snapshot.session_token + && stream.channel.shared_stream_id == snapshot.channel.shared_stream_id + }) + }); + if let Some(current) = current { + self.event_manager + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(current.clone()))); + } + } + pub(super) async fn emit_promotion_update(&self, username: &str, action: PromotionAction) { let maybe_stream = { let user_connections = self.connections.read().await; @@ -167,7 +224,7 @@ impl ActiveUserManager { ); } } - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.emit_current_stream_update(&stream_info).await; } } diff --git a/backend/session/src/active_user_manager/tests.rs b/backend/session/src/active_user_manager/tests.rs index 8cd943389..e6d945e9d 100644 --- a/backend/session/src/active_user_manager/tests.rs +++ b/backend/session/src/active_user_manager/tests.rs @@ -4,6 +4,7 @@ mod adaptive_streams; mod admission; mod expiry_gc; mod observability; +mod provider_handover; mod provider_headers; mod reentry_protection; mod request_claims; diff --git a/backend/session/src/active_user_manager/tests/adaptive_streams.rs b/backend/session/src/active_user_manager/tests/adaptive_streams.rs index 0d045f46f..45a93d54c 100644 --- a/backend/session/src/active_user_manager/tests/adaptive_streams.rs +++ b/backend/session/src/active_user_manager/tests/adaptive_streams.rs @@ -200,7 +200,7 @@ async fn test_kicked_release_does_not_preserve_adaptive_stream() { let removed = manager.release_connection_as_kicked(&addr).await; assert!(removed.addr_removed); assert_eq!(removed.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -323,7 +323,7 @@ async fn test_adaptive_session_release_connection_preserves_logical_stream_and_s assert!(released.removed_streams.is_empty(), "adaptive session should remain logically active"); assert_eq!(manager.user_connections("user1").await, 0); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); let connections = manager.connections.read().await; let preserved_stream = connections @@ -413,7 +413,7 @@ async fn test_release_stream_ignores_preserved_adaptive_entry() { let released = manager.release_connection(&addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); assert!(manager.release_stream(&addr).await.is_none()); } @@ -448,7 +448,7 @@ async fn test_release_stream_without_session_removes_adaptive_stream_instead_of_ let released = manager.release_stream(&addr).await; assert!(released.is_some(), "stream without schedulable expiry must be removed"); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -597,7 +597,7 @@ async fn catchup_release_connection_preserves_logical_stream_until_session_expir ); assert_eq!(manager.user_connections(&user.username).await, 0); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); let connections = manager.connections.read().await; let preserved_stream = connections @@ -895,7 +895,7 @@ async fn preserved_soft_reactivation_and_cleanup_leave_normal_slot_unchanged() { manager.release_stream_by_uid(&normal_addr, normal_stream_uid).await.expect("normal stream should release"); let connections = manager.connections.read().await; let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.is_empty()); + assert_eq!(connection_data.streams.len(), 0); assert!(connection_data.stream_kinds.is_empty()); assert_no_real_connection_slots(connection_data); drop(connections); @@ -1105,7 +1105,7 @@ async fn release_session_streams_notifies_frontend_for_each_removed_row() { vec![(addr, 9101)], "lease-idle release removes the panel row and must tell the frontend" ); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] @@ -1156,7 +1156,7 @@ async fn expire_grace_active_notifies_frontend_for_each_removed_row() { manager.expire_grace_active(&user.username, &token, version).await; assert_eq!(drain_disconnected_streams(&mut events), vec![(addr, 9104)]); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] @@ -1218,7 +1218,7 @@ async fn expire_grace_active_removes_every_row_of_the_expired_session() { let mut removed = drain_disconnected_streams(&mut events); removed.sort_unstable_by_key(|(_, uid)| *uid); assert_eq!(removed, vec![(addr, 9107), (second_addr, 9108)]); - assert!(manager.panel_streams().await.is_empty()); + assert_eq!(manager.panel_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/session/src/active_user_manager/tests/expiry_gc.rs b/backend/session/src/active_user_manager/tests/expiry_gc.rs index db3637605..f043b9a5d 100644 --- a/backend/session/src/active_user_manager/tests/expiry_gc.rs +++ b/backend/session/src/active_user_manager/tests/expiry_gc.rs @@ -145,7 +145,7 @@ async fn test_preserved_adaptive_stream_is_pruned_after_session_ttl() { manager .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) .await; - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -389,7 +389,7 @@ async fn test_due_adaptive_expiry_reschedules_when_session_timestamp_changes() { *expiry_index.get(&key).unwrap() }; assert!(new_expires_at > old_expires_at); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -533,7 +533,7 @@ async fn test_due_adaptive_expiry_does_not_block_on_full_cleanup_channel() { let queued_event = cleanup_rx.try_recv().expect("prefilled cleanup event should remain queued"); assert!(matches!(queued_event, CleanupEvent::ReleaseConnection { .. })); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] diff --git a/backend/session/src/active_user_manager/tests/provider_handover.rs b/backend/session/src/active_user_manager/tests/provider_handover.rs new file mode 100644 index 000000000..0c0d7cad9 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/provider_handover.rs @@ -0,0 +1,321 @@ +use super::{ + test_adaptive_channel, test_user_credentials, ActiveUserConnectionParams, ActiveUserManager, + CreateUserSessionParams, +}; +use crate::{ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{ActiveUserConnectionChange, EventMessage, StreamInfo, UserConnectionPermission}, + utils::Internable, +}; +use std::{borrow::Cow, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint}; + +async fn viewer(manager: &ActiveUserManager, uid: u32, username: &str, shared_id: Option) -> StreamInfo { + let addr = format!("127.0.0.1:{}", 14000 + uid).parse().unwrap(); + let fingerprint = Fingerprint::new(format!("viewer-{uid}"), "127.0.0.1".into(), addr); + let user = test_user_credentials(username, 10, 0); + let token = format!("session-{uid}"); + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: &token, + virtual_id: uid, + provider: "account-a", + stream_url: "http://origin.test/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager.add_connection(&addr).await; + let mut channel = test_adaptive_channel(uid); + channel.shared = shared_id.is_some(); + channel.shared_stream_id = shared_id; + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: uid + 100, + username, + max_connections: 10, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "account-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("handover-test"), + session_token: Some(&token), + }) + .await + .expect("viewer registers") +} + +async fn fixture() -> Arc { + let events = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&Config::default(), &Arc::new(ArcSwapOption::default()), &events)); + viewer(&manager, 1, "alice", Some(42)).await; + viewer(&manager, 2, "bob", Some(42)).await; + viewer(&manager, 3, "alice", Some(42)).await; + manager +} + +async fn project(manager: &ActiveUserManager) -> usize { + manager.lock_shared_stream_provider_projection(42).await.update_provider(&"account-b".intern()) +} + +#[tokio::test] +async fn handover_updates_preserved_viewer_rows() { + let manager = fixture().await; + manager.release_connection(&"127.0.0.1:14002".parse().unwrap()).await; + let before = manager.panel_streams().await; + assert!(before.iter().any(|row| row.preserved)); + assert_eq!(project(&manager).await, 3); + let after = manager.panel_streams().await; + assert_eq!(after.len(), 3); + assert!(after.iter().all(|row| row.provider.as_ref() == "account-b")); + assert_eq!(after.iter().filter(|row| row.preserved).count(), 1); +} + +#[tokio::test] +async fn handover_emits_one_update_per_changed_viewer() { + let manager = fixture().await; + let mut events = manager.events().get_event_channel(); + assert_eq!(project(&manager).await, 3); + assert_eq!(project(&manager).await, 0); + let mut received = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(row)) => received.push(row), + other => panic!("projection must emit only full Updated events, got {other:?}"), + } + } + let mut rows = manager.panel_streams().await; + rows.sort_by_key(|row| row.uid); + received.sort_by_key(|row| row.uid); + assert_eq!(received, rows); +} + +#[tokio::test] +async fn provider_projection_preserves_stream_identity_and_accounting() { + let manager = fixture().await; + manager.release_connection(&"127.0.0.1:14002".parse().unwrap()).await; + let mut expected = manager.panel_streams().await; + let counts = manager.active_users_and_connections().await; + let resources = manager.playback_resource_counts().await; + let sessions = { + let connections = manager.connections.read().await; + connections + .by_key + .iter() + .map(|(user, data)| { + ( + user.clone(), + data.sessions + .iter() + .map(|session| { + ( + session.token.clone(), + session.provider.clone(), + session.ts, + session.started_at, + session.identity(), + session.lifecycle.clone(), + ) + }) + .collect::>(), + ) + }) + .collect::>() + }; + project(&manager).await; + for row in &mut expected { + row.provider = "account-b".intern(); + } + let mut actual = manager.panel_streams().await; + expected.sort_by_key(|row| row.uid); + actual.sort_by_key(|row| row.uid); + assert_eq!(actual, expected); + assert_eq!(manager.active_users_and_connections().await, counts); + assert_eq!(manager.playback_resource_counts().await, resources); + let connections = manager.connections.read().await; + let after = connections + .by_key + .iter() + .map(|(user, data)| { + ( + user.clone(), + data.sessions + .iter() + .map(|session| { + ( + session.token.clone(), + session.provider.clone(), + session.ts, + session.started_at, + session.identity(), + session.lifecycle.clone(), + ) + }) + .collect::>(), + ) + }) + .collect::>(); + assert_eq!(after, sessions); +} + +#[tokio::test] +async fn handover_only_updates_rows_of_the_matching_shared_session() { + let manager = fixture().await; + viewer(&manager, 4, "alice", Some(43)).await; + viewer(&manager, 5, "bob", None).await; + manager.terminate_session("alice", "session-3").await; + let before = manager.panel_streams().await; + assert_eq!(project(&manager).await, 2); + let mut expected = before; + for row in &mut expected { + if row.channel.shared_stream_id == Some(42) { + row.provider = "account-b".intern(); + } + } + let mut actual = manager.panel_streams().await; + expected.sort_by_key(|row| row.uid); + actual.sort_by_key(|row| row.uid); + assert_eq!(actual, expected); + assert!(actual.iter().all(|row| row.session_token.as_deref() != Some("session-3"))); +} + +#[derive(Clone, Copy)] +enum ViewerRelease { + Connection, + Stream, +} + +#[derive(Clone, Copy)] +enum ConcurrentViewerChange { + Handover, + NewerHandover, + Removal, + Reactivation, +} + +async fn reactivate_viewer(manager: &ActiveUserManager, row: &StreamInfo) { + let fingerprint = Fingerprint::new("viewer-2".into(), row.client_ip.clone(), row.addr); + manager + .update_connection(ActiveUserConnectionParams { + uid: 22, + meter_uid: row.meter_uid, + username: &row.username, + max_connections: 10, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: row.provider.clone(), + stream_channel: &row.channel, + user_agent: Cow::Borrowed(&row.user_agent), + session_token: row.session_token.as_deref(), + }) + .await + .expect("viewer reactivates the same row"); +} + +async fn assert_delayed_release_publishes_current_viewer(release_kind: ViewerRelease, change: ConcurrentViewerChange) { + let manager = fixture().await; + if matches!(change, ConcurrentViewerChange::NewerHandover) { + assert_eq!(project(&manager).await, 3); + } + let addr = "127.0.0.1:14002".parse().unwrap(); + let mut events = manager.events().get_event_channel(); + let expiry = manager.adaptive_expiry_index.lock().await; + let mut release = Box::pin(async { + match release_kind { + ViewerRelease::Connection => { + manager.release_connection(&addr).await; + } + ViewerRelease::Stream => { + assert!(matches!( + manager.release_stream_request_by_uid(&addr, 2).await, + super::StreamRequestDetach::Preserved(_) + )); + } + } + }); + assert!(futures::poll!(release.as_mut()).is_pending()); + assert!(manager.panel_streams().await.iter().any(|row| row.uid == 2 && row.preserved)); + let provider = match change { + ConcurrentViewerChange::NewerHandover => "account-c", + ConcurrentViewerChange::Handover | ConcurrentViewerChange::Removal | ConcurrentViewerChange::Reactivation => { + "account-b" + } + }; + assert_eq!(manager.lock_shared_stream_provider_projection(42).await.update_provider(&provider.intern()), 3); + match change { + ConcurrentViewerChange::Handover | ConcurrentViewerChange::NewerHandover => {} + ConcurrentViewerChange::Removal => { + assert!(manager.terminate_session("bob", "session-2").await); + // Any further Updated frame would resurrect the row after its removal event. + events = manager.events().get_event_channel(); + } + ConcurrentViewerChange::Reactivation => { + let row = manager.panel_streams().await.into_iter().find(|row| row.uid == 2).unwrap(); + reactivate_viewer(&manager, &row).await; + } + } + let expected = manager.panel_streams().await.into_iter().find(|row| row.uid == 2); + drop(expiry); + release.await; + let mut delivered = Vec::new(); + while let Ok(event) = events.try_recv() { + if let EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(row)) = event { + if row.uid == 2 { + delivered.push(row); + } + } + } + if let Some(expected) = expected { + assert_eq!( + delivered.last(), + Some(&expected), + "last delivered update must match the complete authoritative row" + ); + assert!(delivered.iter().all(|row| row.provider.as_ref() != "account-a"), "no old account after handover"); + } else { + assert_eq!(delivered, [], "a delayed release must not republish a removed viewer"); + } + assert_eq!(manager.panel_streams().await.iter().filter(|row| row.uid != 2).count(), 2); +} + +#[tokio::test] +async fn delayed_connection_release_cannot_restore_previous_provider() { + assert_delayed_release_publishes_current_viewer(ViewerRelease::Connection, ConcurrentViewerChange::Handover).await; +} + +#[tokio::test] +async fn delayed_stream_release_cannot_restore_previous_provider() { + assert_delayed_release_publishes_current_viewer(ViewerRelease::Stream, ConcurrentViewerChange::Handover).await; +} + +#[tokio::test] +async fn delayed_release_cannot_overwrite_newer_handover() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::NewerHandover).await; + } +} + +#[tokio::test] +async fn delayed_release_does_not_republish_removed_viewer() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::Removal).await; + } +} + +#[tokio::test] +async fn delayed_release_publishes_current_reactivated_viewer() { + for release in [ViewerRelease::Connection, ViewerRelease::Stream] { + assert_delayed_release_publishes_current_viewer(release, ConcurrentViewerChange::Reactivation).await; + } +} diff --git a/backend/session/src/active_user_manager/tests/request_claims.rs b/backend/session/src/active_user_manager/tests/request_claims.rs index 516c079e0..8450fb118 100644 --- a/backend/session/src/active_user_manager/tests/request_claims.rs +++ b/backend/session/src/active_user_manager/tests/request_claims.rs @@ -48,7 +48,7 @@ async fn affine_request_claims_keep_playback_until_oldest_body_finishes() { let removed = manager.release_stream_by_uid(&addr, 41).await; assert!(removed.as_ref().is_some_and(|stream| stream.uid == 41)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -89,7 +89,7 @@ async fn affine_request_claims_keep_playback_until_newest_body_finishes() { let removed = manager.release_stream_by_uid(&addr, 46).await; assert!(removed.as_ref().is_some_and(|stream| stream.uid == 45)); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } fn shared_hls_channel(virtual_id: u32, shared_stream_id: u64) -> shared::model::StreamChannel { diff --git a/backend/session/src/active_user_manager/tests/socket_activity.rs b/backend/session/src/active_user_manager/tests/socket_activity.rs index 0af494fd2..52a61bc4c 100644 --- a/backend/session/src/active_user_manager/tests/socket_activity.rs +++ b/backend/session/src/active_user_manager/tests/socket_activity.rs @@ -284,13 +284,13 @@ async fn kicked_release_removes_preserved_adaptive_stream_without_socket_registr let released = manager.release_connection(&addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(released.removed_streams.len(), 0); + assert_eq!(manager.active_streams().await.len(), 0); let kicked = manager.release_connection_as_kicked(&addr).await; assert!(kicked.addr_removed); assert_eq!(kicked.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -564,7 +564,7 @@ async fn release_stream_by_uid_finds_original_user_after_shared_addr_owner_chang assert!(manager.release_stream_by_uid(&addr, 44).await.is_some()); assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -619,7 +619,7 @@ async fn release_connection_cleans_every_user_stream_for_reused_addr_only() { manager.release_connection(&unrelated_addr).await; assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); + assert_eq!(manager.active_streams().await.len(), 0); } #[tokio::test] @@ -1188,7 +1188,7 @@ async fn vod_session_survives_overlapping_and_seek_sockets() { assert!(manager.release_stream(&range_addr).await.is_none()); let released = manager.release_connection(&range_addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); { let connections = manager.connections.read().await; @@ -1217,7 +1217,7 @@ async fn vod_session_survives_overlapping_and_seek_sockets() { assert!(manager.release_stream(&seek_addr).await.is_none()); let released = manager.release_connection(&seek_addr).await; assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); + assert_eq!(released.removed_streams.len(), 0); let connections = manager.connections.read().await; let connection_data = connections.by_key.get("user1").expect("user connection data"); diff --git a/backend/session/src/connection_manager/cleanup.rs b/backend/session/src/connection_manager/cleanup.rs index 1deb00438..04c6d19a7 100644 --- a/backend/session/src/connection_manager/cleanup.rs +++ b/backend/session/src/connection_manager/cleanup.rs @@ -249,7 +249,7 @@ pub(super) async fn handle_update_detail_and_release_provider( ) .await; } - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + deps.user_manager.emit_current_stream_update(&stream_info).await; } if let Some(handle) = handle { deps.provider_manager.release_handle(&handle); diff --git a/backend/session/src/connection_manager/registration.rs b/backend/session/src/connection_manager/registration.rs index 2a912d907..c8c8fea46 100644 --- a/backend/session/src/connection_manager/registration.rs +++ b/backend/session/src/connection_manager/registration.rs @@ -6,7 +6,7 @@ use super::{ use crate::{uses_direct_body_idle_timeout, ActiveUserConnectionParams}; use log::warn; use shared::{ - model::{ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamInfo}, + model::{CustomVideoStreamType, StreamInfo}, utils::sanitize_sensitive_info, }; use std::{net::SocketAddr, sync::Arc}; @@ -170,8 +170,7 @@ impl ConnectionManager { ); self.touch_direct_body_activity(&fingerprint.addr); } - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + self.user_manager.emit_current_stream_update(&stream_info).await; RegisteredPlaybackRequest::with_cleanup(uid, Some(stream_info), Some(cleanup)) } else { // Registration failed: nothing was inserted, so the cleanup must not fire. @@ -187,13 +186,13 @@ impl ConnectionManager { pub async fn update_stream_detail(&self, addr: &SocketAddr, video_type: CustomVideoStreamType) { if let Some(stream_info) = self.user_manager.update_stream_detail(addr, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.user_manager.emit_current_stream_update(&stream_info).await; } } pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) { if let Some(stream_info) = self.user_manager.update_stream_detail_by_uid(uid, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + self.user_manager.emit_current_stream_update(&stream_info).await; } } } diff --git a/backend/session/src/connection_manager/tests/lifecycle.rs b/backend/session/src/connection_manager/tests/lifecycle.rs index 90c18955f..03668eaa4 100644 --- a/backend/session/src/connection_manager/tests/lifecycle.rs +++ b/backend/session/src/connection_manager/tests/lifecycle.rs @@ -531,7 +531,7 @@ async fn shutdown_releases_active_streams() { }) .await; assert!(stream_info.is_some(), "registration must succeed"); - assert!(!manager.user_manager.active_streams().await.is_empty()); + assert_ne!(manager.user_manager.active_streams().await.len(), 0); manager.shutdown().await; assert!(manager.user_manager.active_streams().await.is_empty(), "shutdown must release active streams"); diff --git a/backend/session/src/event_manager.rs b/backend/session/src/event_manager.rs index 1d202442d..06a5a5646 100644 --- a/backend/session/src/event_manager.rs +++ b/backend/session/src/event_manager.rs @@ -746,7 +746,7 @@ mod snapshot_and_ring_tests { manager.send_event(EventMessage::ConfigChange(ConfigType::Config)); manager.send_event(EventMessage::RecordingChanged); - assert!(manager.snapshot().is_empty()); + assert_eq!(manager.snapshot().len(), 0); } #[test] diff --git a/backend/session/src/event_manager/playlist_update.rs b/backend/session/src/event_manager/playlist_update.rs index a293d5ee3..d42647ccf 100644 --- a/backend/session/src/event_manager/playlist_update.rs +++ b/backend/session/src/event_manager/playlist_update.rs @@ -114,7 +114,7 @@ mod tests { manager.send_event(completed("r1", 2)); assert_eq!(manager.playlist_update_snapshot(), vec![own]); manager.send_event(completed("r1", 1)); - assert!(manager.playlist_update_snapshot().is_empty()); + assert_eq!(manager.playlist_update_snapshot().len(), 0); manager.send_event(EventMessage::PlaylistUpdateProgress(progress("r1", 1, 7))); assert!(manager.playlist_update_snapshot().is_empty(), "late progress cannot resurrect a finished input/run"); let next = progress("r2", 2, 7); @@ -184,7 +184,7 @@ mod tests { for event in [PlaylistUpdateProgressEvent::global("target", "working"), no_order, no_run] { manager.send_event(EventMessage::PlaylistUpdateProgress(event)); } - assert!(manager.playlist_update_snapshot().is_empty()); + assert_eq!(manager.playlist_update_snapshot().len(), 0); manager.send_event(EventMessage::PlaylistUpdateProgress(valid.clone())); manager.send_event(EventMessage::PlaylistUpdateProgress(progress("conflicting", 1, 7))); assert_eq!(manager.playlist_update_snapshot(), vec![valid]); diff --git a/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs index fd11d8588..f0a48acda 100644 --- a/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs +++ b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs @@ -221,7 +221,7 @@ async fn shared_origin_abort_before_user_registration_releases_physical_and_logi .await?; assert_eq!(providers.get_provider_connections_count(), 0); assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); - assert!(users.active_streams().await.is_empty()); + assert_eq!(users.active_streams().await.len(), 0); Ok(()) } @@ -414,7 +414,7 @@ async fn shutdown_with_unpolled_shared_body_leaves_no_requests_or_subscribers() connections.shutdown().await; assert_eq!(providers.get_provider_connections_count(), 0); - assert!(users.active_streams().await.is_empty()); + assert_eq!(users.active_streams().await.len(), 0); assert!(manager.get_shared_state(url).await.is_none()); drop(unpolled_body); diff --git a/docs/src/configuration/hls-cache-state-machine.md b/docs/src/configuration/hls-cache-state-machine.md index ef27b40dc..ae0b1f55f 100644 --- a/docs/src/configuration/hls-cache-state-machine.md +++ b/docs/src/configuration/hls-cache-state-machine.md @@ -304,7 +304,7 @@ the old location. | Target duration fallback | 15 seconds | Used before an origin target duration is known. | | Publication lateness | `1.5 * target_duration` | Observation signal only; never a cutover or terminal deadline. | | Recovery trigger | Lease READY reserve minus full acceptance ETA and transition margin | Starts recovery while the configured burst can still complete. | -| Terminal cutover | Lease-specific safe commit deadline | Prevents terminal publication after the lease's usable reserve is exhausted. | +| Terminal cutover | Lease-specific safe splice deadline | Bounds a live-tail splice; a late finite Custom Response uses a separate bounded commit budget. | | HLS cache GC interval | 30 seconds | Periodic cleanup cadence for cache objects and stale runtime state. | | Temporary file retention | 30 seconds | Retention window for temporary HLS cache files. | | Failed segment retention | 10 seconds | Short retention for failed segment state. | diff --git a/docs/src/configuration/reverse-proxy.md b/docs/src/configuration/reverse-proxy.md index ed9e3a64a..bd6953dde 100644 --- a/docs/src/configuration/reverse-proxy.md +++ b/docs/src/configuration/reverse-proxy.md @@ -290,6 +290,12 @@ For the configuration reference, see [Shared HLS Configuration](./shared-hls-con For the shared session, access lease, and transient delivery state machines, see [HLS Cache State Machines](./hls-cache-state-machine.md). +The session and global byte limits cover committed objects, immutable revisions, and outstanding write reservations. +Staging and postprocessing grow their budgets before writing additional bytes. Committing an object consumes its own +reservation; capacity released by a completed or aborted write wakes deferred downloads. Both targeted reclamation +and periodic GC retire obsolete, unpinned revisions. See +[capacity diagnostics](./shared-hls-troubleshooting.md#symptom-downloads-wait-for-local-cache-capacity). + ```yaml reverse_proxy: hls_cache: @@ -316,6 +322,15 @@ reverse_proxy: max_parallel_jobs: 1 ``` +Short rolling origin playlists (one or two segments) are permitted only after the session evaluates its complete +configured manifest recovery burst. A suitable window with at least three segments takes priority. The resulting +permission belongs to the session's validated origin binding and epoch, and survives subsequent short manifests +and new access leases. Freshness refreshes share the existing singleflight; they do not themselves renew the burst. +Failed qualification also shares its retry receipt and backoff. A genuine origin revalidation requires a new +qualification. One warning per short-window qualification cites RFC 8216 section 6.2.2; larger available cache +windows, media readiness, lease validity, and resource retention rules still apply. See +[Shared HLS configuration](shared-hls-configuration.md#unstable-manifests). + ### HLS Cache Parameter Details | Parameter | Type | Default | Technical Impact | @@ -351,7 +366,20 @@ Important boundaries: * `reverse_proxy.hls_cache` only prepares the global cache engine. A target must also set `options.share_live_streams.hls: true` in `source.yml` before generated HLS live entries use the shared HLS path. -* HLS cache retry behavior is fixed internally and is not user-configurable. +* HLS cache retry behavior is fixed internally and is not user-configurable. Shared HLS origin-manifest retries cap an + upstream `Retry-After` delay at half the last known target duration. When no target duration is available, the + 15-second fallback limits the delay to 7.5 seconds. +* Playback reserve uses a stable session media clock and completed segment deliveries. Removing old cache entries + does not reset the playback position; missing segments, MAPs and keys still limit the usable reserve. +* A retryable manifest failure marks the origin as degraded on the first failed attempt. A bounded background owner + reevaluates the remaining reserve and schedules the terminal transition independently of origin retries and client + playlist polling. New origin progress invalidates the pending transition. +* A terminal video splice must commit before its safe splice deadline. After that deadline, a valid published lease + receives an HTTP `200` finite `channel_unavailable` Custom Response on the same manifest URL, containing only prepared + custom segments. This fallback needs no old cache segments and has a separate bounded commit budget that playlist + polls cannot renew. Successful authorized custom-segment requests renew the lease and session activity independently + of manifest polling after `ENDLIST`. Disabled, missing or invalid custom media can still produce an error response; + concrete preparation failures such as `TargetDurationExceeded` retain their diagnostic reason. * `reverse_proxy.rewrite_secret` must stay stable. It is used for future HLS `proxy_session_id` values and transient resource IDs. * `session_idle_timeout` controls HLS cache access-lease validity and idle cleanup. It is independent from `reverse_proxy.stream.hls_session_ttl_secs`, which belongs to the non-cache HLS request continuity path. @@ -590,6 +618,24 @@ reverse_proxy: ## Additional Information +### Shared HLS account replacement + +A shared HLS session checks its bound account when it needs origin work. Removed, expired, disabled and runtime-blocked +accounts are replaced using available accounts from the same input pool. Manifest refreshes, segment and MAP cache misses, +key/transient fetches and existing background workers use the same preparation path. Cached media needs no new allocation. +There is no additional monitoring service or configuration setting. + +Replacement retains the shared session identity, valid access leases, published timeline and cached media. Tuliprox drops +old provider cookies and recovery targets, resolves the entry URL with the replacement account, and qualifies its manifest +against the existing timeline and content/host checks before scheduling new media. Catchup retains its archive identity. +Account replacement assigns new resource IDs even when the concrete key URI is unchanged. Cached AES keys for previously +published media retain their old IDs, so the replacement manifest cannot reuse a key cached for the previous account. + +If no suitable account is available, requests receive the existing temporary response with `Retry-After`; background work +uses its existing retry budgets. Failed replacement attempts are separated by at least two seconds. Full capacity, foreign +reservations, timeouts and isolated HTTP errors do not invalidate an otherwise healthy account. Replacement never borrows +an account from another source or guarantees uninterrupted playback. + ### Session TTLs for HLS (`.m3u8`) & Catchup HLS streams do not consist of an endless TCP pipe. Instead, the player downloads small `.ts` segments every few seconds (e.g., `seg1.ts`, `seg2.ts`). diff --git a/docs/src/configuration/shared-hls-configuration.md b/docs/src/configuration/shared-hls-configuration.md index 869539f5c..d963f259d 100644 --- a/docs/src/configuration/shared-hls-configuration.md +++ b/docs/src/configuration/shared-hls-configuration.md @@ -197,6 +197,21 @@ reverse_proxy: Use recovery burst levels gradually. Start with `friendly`. Move to stronger levels only when logs show manifest refresh instability and the provider can tolerate additional requests. +Shared HLS evaluates a short live origin window (one or two segments) with the complete configured burst before +allowing it. A suitable candidate with at least three segments takes priority. If only suitable short windows +remain, the session records permission to publish them and emits one warning referencing RFC 8216 section 6.2.2. +The `off` level evaluates its single candidate; no additional burst level is required. + +All access leases share the qualification and refresh work, including failure backoff. New leases, ordinary +sequence movement, and later changes such as one → three → one segments do not consume or renew the permission. +A request that needs a fresher manifest uses the normal shared refresh. A changed origin binding or epoch, +acceptance conflict, or lifecycle state requiring origin revalidation starts a new shared qualification. + +The proxy continues to use larger available cache windows. Strip keeps already short windows intact, and +startup admission and reserve accounting use their actual media duration. Missing segments, maps or keys, +replayed media, empty playlists, and invalid leases remain subject to the existing checks. This exception cannot +create media progress when the origin stops publishing. + ### Problematic MPEG-TS segments ```yaml @@ -290,8 +305,9 @@ or silently change the bytes of an already published URI. Revision and repair pe disk cache limits. Conservative behavior remains the default; existing sessions keep their startup policy after reload. Restart after changing to `conservative` when an immediate rollback of running sessions is required. -Existing provider and fetch concurrency limits remain in effect. The initial playlist needs at least three -publishable segments; `optimistic` counts successors the origin has accepted. With fewer than three concurrent +Existing provider and fetch concurrency limits remain in effect. The initial playlist normally needs at least three +publishable segments; a session qualified for short origin windows can publish one or two. `optimistic` counts +successors the origin has accepted. With fewer than three concurrent fills, a cold session can still wait for an earlier origin EOF before that window is available. Raising the existing per-session concurrency limit to three is an operator choice and must fit the provider's connection allowance; startup mode never raises it automatically. diff --git a/docs/src/configuration/shared-hls-runtime-flow.md b/docs/src/configuration/shared-hls-runtime-flow.md index 69d67b2c5..f954a1308 100644 --- a/docs/src/configuration/shared-hls-runtime-flow.md +++ b/docs/src/configuration/shared-hls-runtime-flow.md @@ -211,13 +211,13 @@ runs the parser or writer. The prepared-bundle key binds: One ready bundle contains twelve small immutable terminal blocks. For block index `x`, timestamp progression is: ```text -offset_90khz(x) = x * target_duration_ms * 90 +offset_90khz(x) = x * asset_duration_ticks_90khz offset_pcr(x) = offset_90khz(x) * 300 ``` -The target duration is intentionally the timestamp stride; the measured terminal asset duration is the `#EXTINF` value. -When that asset is shorter than the target duration, the resulting bounded timestamp gap is intentional. There is no old -`index * asset_duration_ticks` terminal calculation. +The measured terminal asset duration determines the timestamp stride and the `#EXTINF` value. The target duration is +retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments. Both the +live-tail splice and the late Custom Response use this same prepared bundle. ## Warm terminal-tail response @@ -234,7 +234,27 @@ the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment r /hls/shared/live///terminal//.ts ``` -Only that lease- and generation-bound route serves prepared terminal bytes and finite full/range responses. Existing +If the safe splice deadline has already elapsed, a valid lease with a previously published manifest instead commits a +finite Custom Response without a live suffix. Its canonical manifest returns HTTP `200` with no redirect, twelve +prepared `channel_unavailable` segments under `/cvs/hls///.ts`, a clear-key +reset, a discontinuity and `#EXT-X-ENDLIST`. The target duration is retained; media sequence starts after the highest +segment number previously published to that lease, and discontinuity sequence does not decrease. No live cache base, +origin request or timestamp rewrite is needed to serve these custom segments. Playback may stall before this late +warning becomes visible: the fallback does not promise a seamless transition after the deadline. + +The late commit has one bounded retry budget independent of the expired splice deadline. Repeated playlist requests +cannot extend it. Lease incarnation, publication/cursor generation, origin recovery and configured asset revision +are revalidated at commit. The frozen manifest and media remain unchanged on replay and asset reload; access still +requires the valid lease and existing user authorization. Successful authorized GET, HEAD and range requests for the +late custom segments activate or renew that exact lease and refresh session activity using the configured +`session_idle_timeout`; playback does not depend on further manifest polls after `#EXT-X-ENDLIST`. Invalid routes, +unsatisfiable ranges and denied authorization do not renew the lease, and an expired lease cannot be reactivated. +Disabled, missing or incompatible custom media can retain an error response. Preparation failures retain their typed +compatibility reason, including `TargetDurationExceeded` with the measured asset and playlist target durations. +The original compatible live-tail splice continues to apply before its deadline. + +Only the lease- and generation-bound terminal route serves the live-tail splice's prepared terminal bytes and finite +full/range responses. Existing normal segment, MAP, and `/r` URLs retain their original meaning and never redirect to a manifest or substitute slate bytes. Terminal playback mode is sticky for the lease; later shared-session recovery can serve other or new leases but cannot reactivate the terminal lease. @@ -244,6 +264,36 @@ cannot reactivate the terminal lease. The shared session can hold an origin account binding. That binding represents the provider account or provider connection Tuliprox uses for origin work. +Before necessary origin I/O, a session advances overlap promotion, detachment and reclaim, then re-reads the current input +configuration and binding. Account replacement is serialized per session. Missing, expired and blocked accounts share one replacement path; +blocked includes disabled accounts and runtime +health observations before persistence. Allocation stays inside the original input and follows existing capacity and +priority rules. An absent input cannot migrate to another source. A failed attempt retains its two-second backoff even +when the binding has become detached. + +The HLS worker boundary uses a typed future-returning prepare callback. Its app adapter holds a weak application-state +reference. Manifest, segment, MAP, key/transient and existing background work prepare before capturing a fetch generation +or snapshot. Only authorized live work may prepare; terminal/expired leases and cache hits do not allocate. +Preparation returns the resolved entry together with its account identity, work generation and provider headers. Refresh +initialization carries that result intact; a changed binding or generation requires preparation again or rejects the start. +An unused allocation for the previous entry is released before preparing its replacement. + +An account handoff advances account/work generations, invalidates waiting origin work, old cookies, recovery targets and +unfetched resource bindings, and keeps session identity, access leases, archive identity, timeline and cached objects. +Media work first requests manifest qualification through the existing refresh flow. The first request resolves the new +entry, including `provider://` entries; any subsequent recovery is bound exclusively to that new concrete HTTP(S) target. +Continuity, content-anchor and cross-host checks still apply. Old in-flight results and tagged cleanup cannot publish over +or release a successor binding. After releasing an old worker permit, the existing scheduler resumes qualified successor +work, including when only one global fetch permit exists. Allocation handles own their release responsibility across +asynchronous preparation. +Each refresh also owns a separate singleflight token. An invalidated refresh releases its own flag even after a failed +replacement, while a late completion cannot clear a successor token, including one started in the same millisecond. + +Account handoff gives newly published transient resources and normal AES key references a separate resource-ID namespace. +Identical origin key URLs therefore cannot inherit cached keys from the previous account. Old published references keep +their original mappings and cached bytes; existing resource TTLs and metadata limits still govern retention. Rolling EVENT +rewrites restart at handoff and preserve the new namespace when appending subsequent manifest lines. + Origin account protection is based on successful media access: | Protection state | Meaning | diff --git a/docs/src/configuration/shared-hls-troubleshooting.md b/docs/src/configuration/shared-hls-troubleshooting.md index ea945fabd..6b928aab9 100644 --- a/docs/src/configuration/shared-hls-troubleshooting.md +++ b/docs/src/configuration/shared-hls-troubleshooting.md @@ -90,7 +90,11 @@ Fix: For the warm Shared HLS path, this is a lease-local terminal decision rather than a failure-counter threshold. The canonical manifest stays on the same URL and returns an HTTP `200` finite playlist when a compatible terminal tail is -ready. An incompatible MAP/key/container/track state fails closed with `503`; it never creates an unsafe TS splice. +ready. Before the splice deadline, an incompatible MAP/key/container/track state can fail closed with `503`. +After the deadline, a valid published lease can receive a finite Custom Response containing only prepared +`channel_unavailable` media, even when its live cache suffix is gone. The playlist uses `/cvs/hls/` URLs and does not +attempt a late TS splice. The player may stall before displaying it. Disabled, missing or invalid custom media, +expired authorization, and exhausted commit capacity can still prevent a usable response. Possible causes: @@ -187,6 +191,38 @@ Fix: - Lower `cache_duration` if unprotected old objects stay too long. - Put `cache_path` on a disk with enough free space and predictable cleanup behavior. +## Symptom: downloads wait for local cache capacity + +`LocalCacheCapacity` means the decoded body cannot fit the current session or global budget. The budget includes +committed segments, MAPs, transient objects, immutable revisions, and the remaining reservations of active writes. +Startup reserves the existing peak allowance (three body copies for processed revisions, five for raw revisions). +Committing a file transfers its bytes from that write's reservation into committed usage; it does not add the same +bytes twice. Unknown or growing bodies reserve additional space before writing it. +Replacing an object with a smaller file releases the old bytes only after the atomic rename succeeds. + +Use these debug messages to distinguish the causes: + +| Log signal | Meaning | +| :--- | :--- | +| `HLS capacity inventory` | Per-session `revision_bytes` and `remaining_reserved_bytes`, plus revision deletion outcomes. | +| `HLS capacity FIFO` | The oldest segment, its status, and its actual blocker: protection, pending work, or no candidates. | +| `outcome=lock-conflict protected_working_set_known=false` | Selection could not acquire the session lock. A zero protection estimate is unknown, not evidence that every byte is reclaimable. | +| `HLS revision GC` | Bounded deletion attempts: `planned`, `succeeded`, `failed`, and actual `reclaimed_bytes`. | +| `HLS revision delete deferred` | Filesystem error; the revision stays queued for a later attempt and contributes no reclaimed bytes. | + +Both periodic GC and targeted admission release obsolete references and retire unpinned revisions. A protected FIFO +head still stops segment eviction. An inactive `CapacityDeferred` head can be retired once its origin binding and +all protections are gone. Open readers and retained lease publications keep their revision bytes pinned. + +A capacity waiter resumes when reservation, accounting, or playback protection changes, then checks the complete +peak of a fresh attempt before downloading again. Releasing too little capacity does not start another origin fetch. +It does not need a periodic GC tick or traffic from another session. Published revision bytes remain immutable across +repeated requests and ranges, even when the mutable normal cache object is replaced or rolled back. + +The in-memory `revision_disk_bytes` gauge is updated by successful file mutations and inventory reconciliation. +Revision deletion counters distinguish attempts, successes, failures, and bytes actually reclaimed. Inspect sustained +I/O failures or retained readers before changing limits. Do not copy credential-bearing origin URLs into diagnostics. + ## Symptom: cache path permission errors The Tuliprox process must be able to create directories, write temporary files, rename committed cache objects, read cached diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index a3ffe5483..3788a3ba0 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -9,7 +9,7 @@ workspace = true [dependencies] blake3.workspace = true -shared = { version = "3.3.155", path = "../shared" } +shared = { version = "3.3.156", path = "../shared" } chrono.workspace = true yew = "0.23.0" yew-router = "0.20.0" diff --git a/frontend/src/app/components/playlist/epg_view/mod.rs b/frontend/src/app/components/playlist/epg_view/mod.rs index 100535104..df100aed0 100644 --- a/frontend/src/app/components/playlist/epg_view/mod.rs +++ b/frontend/src/app/components/playlist/epg_view/mod.rs @@ -1342,6 +1342,6 @@ mod tests { #[test] fn pending_program_source_id_does_not_treat_numeric_epg_id_as_playlist_id() { let full = EpgGridModel::from_epg_tv(&EpgTv::new(vec![EpgChannel::new("123".intern())]), |_| 0); - assert!(pending_program_source_id(&full.channels[0]).is_empty()); + assert_eq!(pending_program_source_id(&full.channels[0]).len(), 0); } } diff --git a/frontend/src/app/components/playlist/input_update_card.rs b/frontend/src/app/components/playlist/input_update_card.rs index 518ec6d9f..7860a9a7c 100644 --- a/frontend/src/app/components/playlist/input_update_card.rs +++ b/frontend/src/app/components/playlist/input_update_card.rs @@ -1812,7 +1812,7 @@ mod tests { TargetSelectionState::All ); let cleared = reduce(individually_restored, InputUpdateCardAction::ToggleAll(targets.clone())); - assert!(cleared.controls.selected_target_ids.is_empty()); + assert_eq!(cleared.controls.selected_target_ids.len(), 0); assert_eq!( target_selection_state(&cleared.controls.selected_target_ids, &targets), TargetSelectionState::Empty @@ -2260,9 +2260,9 @@ mod tests { let replacement = card(7, &[4]); let targetless = card(7, &[]); - assert!(selected_current_target_ids(&replacement, &state.controls.selected_target_ids).is_empty()); + assert_eq!(selected_current_target_ids(&replacement, &state.controls.selected_target_ids).len(), 0); assert!(!can_start_update(&state, &replacement, true)); - assert!(selected_current_target_ids(&targetless, &state.controls.selected_target_ids).is_empty()); + assert_eq!(selected_current_target_ids(&targetless, &state.controls.selected_target_ids).len(), 0); assert!(!can_start_update(&state, &targetless, true)); } diff --git a/frontend/src/app/components/playlist/playlist_update_view.rs b/frontend/src/app/components/playlist/playlist_update_view.rs index 344be35b0..f9c219d3b 100644 --- a/frontend/src/app/components/playlist/playlist_update_view.rs +++ b/frontend/src/app/components/playlist/playlist_update_view.rs @@ -579,7 +579,7 @@ mod tests { assert_eq!(state.for_input(input_id).details(), [detail]); } assert_eq!(state.for_input(9).status(), InputUpdateCardStatus::Ready); - assert!(state.for_input(9).details().is_empty()); + assert_eq!(state.for_input(9).details().len(), 0); let library = LibraryScanProgressEvent { summary: LibraryScanSummary { status: LibraryScanSummaryStatus::Success, diff --git a/frontend/src/app/components/source_editor/output_xtream_form.rs b/frontend/src/app/components/source_editor/output_xtream_form.rs index d7c9a8fbb..bb9bfa4c3 100644 --- a/frontend/src/app/components/source_editor/output_xtream_form.rs +++ b/frontend/src/app/components/source_editor/output_xtream_form.rs @@ -833,7 +833,7 @@ mod tests { .expect("lists-only trakt config"); assert_eq!(result.lists, lists); - assert!(result.charts.is_empty()); + assert_eq!(result.charts.len(), 0); } #[test] @@ -859,7 +859,7 @@ mod tests { assert_eq!(result.catalog_selection, TraktCatalogSelection::Curated); assert!(!result.include_xtream_base_categories); - assert!(result.lists.is_empty()); + assert_eq!(result.lists.len(), 0); assert_eq!(result.charts, charts); } diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index 3f2717651..6c0b1a4b7 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -205,7 +205,6 @@ fn replace_server_status_snapshot( // Trust backend `panel_streams` as authoritative. Re-merging sticky FE ghosts resurrected // ended archive/HLS rows until a full page reload. dedupe_streams_by_identity(&mut server_status.active_user_streams); - prune_zapped_preserved_streams(&mut server_status.active_user_streams); merge_aux_streams(&mut server_status, download_streams); let server_status = Rc::new(server_status); *status_holder = Some(Rc::clone(&server_status)); @@ -256,14 +255,22 @@ fn apply_active_user_change(server_status: &mut StatusCheck, event: ActiveUserCo } return; } - if let Some(pos) = find_stream_update_index(&server_status.active_user_streams, &stream_info) { - server_status.active_user_streams[pos] = stream_info; - } else { - server_status.active_user_streams.push(stream_info); - } + let provider_only_update = + if let Some(pos) = find_stream_update_index(&server_status.active_user_streams, &stream_info) { + let mut previous = server_status.active_user_streams[pos].clone(); + previous.provider.clone_from(&stream_info.provider); + let provider_only_update = previous == stream_info; + server_status.active_user_streams[pos] = stream_info; + provider_only_update + } else { + server_status.active_user_streams.push(stream_info); + false + }; dedupe_streams_by_identity(&mut server_status.active_user_streams); // Channel zap: new active channel replaces preserved soft-kept previous channel immediately. - prune_zapped_preserved_streams(&mut server_status.active_user_streams); + if !provider_only_update { + prune_zapped_preserved_streams(&mut server_status.active_user_streams); + } } ActiveUserConnectionChange::Disconnected(addr) => { let mut retained = Vec::with_capacity(server_status.active_user_streams.len()); @@ -502,6 +509,58 @@ mod tests { stream } + #[test] + fn provider_update_replaces_existing_shared_hls_rows() { + let mut viewers = vec![ + test_shared_hls_stream(11, "127.0.0.1:11001", Some("viewer-a")), + test_shared_hls_stream(12, "127.0.0.1:11002", Some("viewer-b")), + test_shared_hls_stream(13, "127.0.0.1:11003", Some("viewer-c")), + ]; + viewers[1].preserved = true; + for row in &mut viewers { + row.channel.shared_stream_id = Some(42); + row.provider = "account-a".intern(); + row.started_at = 123; + row.ts = 456; + } + let mut status = shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() }; + let viewer_count = viewers.len(); + for row in &mut viewers { + row.provider = "account-b".intern(); + apply_active_user_change(&mut status, ActiveUserConnectionChange::Updated(row.clone())); + assert_eq!( + status.active_user_streams.len(), + viewer_count, + "provider updates must never prune another viewer" + ); + } + assert_eq!(status.active_user_streams, viewers); + } + + #[test] + fn status_snapshot_replaces_stale_provider_names() { + let mut viewers = vec![ + test_shared_hls_stream(11, "127.0.0.1:11001", Some("viewer-a")), + test_shared_hls_stream(12, "127.0.0.1:11002", Some("viewer-b")), + ]; + viewers[1].preserved = true; + for row in &mut viewers { + row.provider = "account-a".intern(); + } + let mut holder = + Some(Rc::new(shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() })); + for row in &mut viewers { + row.provider = "account-b".intern(); + } + let result = replace_server_status_snapshot( + &mut holder, + shared::model::StatusCheck { active_user_streams: viewers.clone(), ..Default::default() }, + &[], + ); + assert_eq!(result.active_user_streams, viewers); + assert_eq!(holder.unwrap().active_user_streams, viewers); + } + #[test] fn test_find_stream_update_index_prefers_adaptive_session_token_over_addr() { let existing = test_stream(1, "127.0.0.1:1234", Some("tok-hls"), PlaylistItemType::LiveHls); diff --git a/frontend/src/model/input_update_capabilities.rs b/frontend/src/model/input_update_capabilities.rs index 51e342003..a674742e0 100644 --- a/frontend/src/model/input_update_capabilities.rs +++ b/frontend/src/model/input_update_capabilities.rs @@ -79,7 +79,7 @@ mod tests { #[test] fn input_update_capabilities_rescan_is_not_a_provider_policy_and_uses_targets() { let capability = InputUpdateCapabilities::Rescan; - assert!(capability.policies().is_empty()); + assert_eq!(capability.policies().len(), 0); assert!(capability.rebuilds_targets()); assert_eq!(capability.selected_action(InputRefreshPolicy::NORMAL), InputUpdateAction::Rescan); for unavailable in [ diff --git a/frontend/src/model/input_update_card_model.rs b/frontend/src/model/input_update_card_model.rs index e6ca9b1da..e3ecbf70b 100644 --- a/frontend/src/model/input_update_card_model.rs +++ b/frontend/src/model/input_update_card_model.rs @@ -221,7 +221,7 @@ mod tests { let cards = build_input_update_card_models(&sources, &status); let card = cards.iter().find(|card| card.input_id == 40).expect("card without targets"); - assert!(card.targets.is_empty()); + assert_eq!(card.targets.len(), 0); } #[test] @@ -237,7 +237,7 @@ mod tests { assert_eq!(cards.len(), 1); assert_eq!(cards[0].input_id, 7); assert_eq!(cards[0].input_type, InputType::Xtream); - assert!(cards[0].targets.is_empty()); + assert_eq!(cards[0].targets.len(), 0); } #[test] diff --git a/frontend/src/model/pipeline_transparency_model.rs b/frontend/src/model/pipeline_transparency_model.rs index 62e2405ed..1d9930ded 100644 --- a/frontend/src/model/pipeline_transparency_model.rs +++ b/frontend/src/model/pipeline_transparency_model.rs @@ -486,7 +486,7 @@ mod tests { assert_eq!(reloaded.overall_state, InputUpdateCardStatus::Ready); assert_eq!(reloaded.run_id, None); assert_eq!(reloaded.cache_source, None); - assert!(reloaded.cluster_results.is_empty()); + assert_eq!(reloaded.cluster_results.len(), 0); } } } @@ -601,7 +601,7 @@ mod tests { assert_eq!(view.cache_source, None); assert_eq!(view.refresh_policy, None); assert_eq!(view.library_scan_result, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert_eq!(view.cluster_results.is_empty(), !(input_type.is_xtream() || input_type.is_stalker())); assert!(view.cluster_results.iter().all(|cluster| cluster.outcome.is_none() && cluster.quality.is_none() @@ -713,7 +713,7 @@ mod tests { let view = build_input_update_run_views(&sources, &cards, &reloaded, &persisted).remove(0); assert_eq!(view.input_state, InputUpdateCardStatus::Success); assert_eq!(view.run_id, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert_eq!(view.refresh_policy, None); } } @@ -762,7 +762,7 @@ mod tests { let reload = project(&fresh); assert_eq!(reload.library_catalog, Some(catalog.clone())); assert_eq!(reload.library_scan_result, None); - assert!(reload.cluster_results.is_empty()); + assert_eq!(reload.cluster_results.len(), 0); assert_eq!(reload.last_update_at, Some(1234)); let state = fresh .reduce(accepted("r1")) @@ -789,7 +789,7 @@ mod tests { assert_eq!(queued.run_id, Some("r2".into())); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); assert_eq!(queued.library_scan_result, None); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert_eq!(project(&state.clone().reduce(accepted("r1"))), queued); let state = state.reduce(progress(PlaylistUpdateProgressEvent::for_run_input( "r2".into(), @@ -1533,7 +1533,7 @@ mod tests { assert_eq!(view.refresh_policy, None); assert_eq!(view.cache_source, None); assert_eq!(view.last_update_at, None); - assert!(view.progress_details.is_empty()); + assert_eq!(view.progress_details.len(), 0); assert!(view.cluster_results.iter().all(|cluster| cluster.threshold.is_none())); assert_eq!(view.target_results[0].target_id, 99); assert_eq!(view.target_results[0].pipeline, None); @@ -1702,7 +1702,7 @@ mod tests { assert_eq!(queued.run_id.as_ref().map(PlaylistUpdateRunId::as_ref), Some("force-run")); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); assert_eq!(queued.refresh_policy, Some(InputRefreshPolicy::FORCE)); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert_eq!(queued.target_results, partial.target_results); let live = queued.cluster_results[0]; assert_eq!(live.requested, None); diff --git a/frontend/src/model/playlist_update_card_status.rs b/frontend/src/model/playlist_update_card_status.rs index 3b8689df6..c9a184692 100644 --- a/frontend/src/model/playlist_update_card_status.rs +++ b/frontend/src/model/playlist_update_card_status.rs @@ -990,7 +990,7 @@ mod tests { playlist_update_bulk_scope(&sources, &cards), PlaylistUpdateBulkScope { can_submit: true, visible_input_ids: vec![7] } ); - assert!(local_controls.selected_target_ids.is_empty()); + assert_eq!(local_controls.selected_target_ids.len(), 0); assert_eq!(local_controls.policy, InputRefreshPolicy::FORCE); } @@ -1315,7 +1315,7 @@ mod tests { assert_eq!(queued.status(), InputUpdateCardStatus::Queued); assert_eq!(queued.run_id().map(PlaylistUpdateRunId::as_ref), Some("r2")); assert_eq!(queued.refresh_policy(), Some(InputRefreshPolicy::FORCE)); - assert!(queued.details().is_empty()); + assert_eq!(queued.details().len(), 0); assert_eq!(queued.input_telemetry(), None); } @@ -1425,7 +1425,7 @@ mod tests { let runtime = state.for_input(7); assert_eq!(runtime.status(), InputUpdateCardStatus::Failed); - assert!(runtime.details().is_empty()); + assert_eq!(runtime.details().len(), 0); let terminal = state.terminal_by_input_id.get(&7).expect("terminal state"); assert_eq!(terminal.run_id, run_id("r2")); assert_eq!(terminal.execution_order, execution_order(2)); diff --git a/frontend/src/model/staged_completion_tests.rs b/frontend/src/model/staged_completion_tests.rs index 1d9bd9d2f..7f19496bc 100644 --- a/frontend/src/model/staged_completion_tests.rs +++ b/frontend/src/model/staged_completion_tests.rs @@ -88,7 +88,7 @@ fn playlist_update_status_pipeline_transparency_reload_restores_running_target_r let queued = build_input_update_run_views(&sources, &cards, &state, &status).remove(0); assert_eq!(queued.run_id, Some("follow-up".into())); assert_eq!(queued.overall_state, InputUpdateCardStatus::Queued); - assert!(queued.progress_details.is_empty()); + assert_eq!(queued.progress_details.len(), 0); assert!(state.for_input(8).input_telemetry().is_none()); } } @@ -191,7 +191,7 @@ fn playlist_update_status_pipeline_transparency_real_staged_trace_keeps_own_resu "persisted input result proves no target/run result" ); assert_eq!(reload.run_id, None); - assert!(reload.progress_details.is_empty()); + assert_eq!(reload.progress_details.len(), 0); } } diff --git a/frontend/src/services/playlist_service.rs b/frontend/src/services/playlist_service.rs index 036ea2776..4867a72e7 100644 --- a/frontend/src/services/playlist_service.rs +++ b/frontend/src/services/playlist_service.rs @@ -312,7 +312,7 @@ mod tests { assert_eq!(request.target_ids, Some(vec![20, 21])); assert_eq!(request.input_action, Some(action)); assert_eq!(request.input_refresh, None); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(build_playlist_update_bulk_request().input_action, None); } @@ -334,7 +334,7 @@ mod tests { for &policy in InputUpdateCapabilities::for_input(&input).policies() { let request = build_input_playlist_update_request(&[30, 20], InputRefreshOverride { input_id: input.id, policy }); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(vec![30, 20])); assert_eq!(request.input_refresh, Some(InputRefreshOverride { input_id: 7, policy })); } @@ -406,7 +406,7 @@ mod tests { InputRefreshOverride { input_id: 7, policy: InputRefreshPolicy::FORCE }, ); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(vec![30, 20])); assert_eq!( request.input_refresh, @@ -422,7 +422,7 @@ mod tests { InputRefreshOverride { input_id: 7, policy: InputRefreshPolicy::NORMAL }, ); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, Some(expected_ids)); } } @@ -440,7 +440,7 @@ mod tests { fn playlist_update_bulk_request_uses_normal_policy_and_complete_server_target_selection() { let request = build_playlist_update_bulk_request(); - assert!(request.targets.is_empty()); + assert_eq!(request.targets.len(), 0); assert_eq!(request.target_ids, None); assert_eq!(request.input_refresh, None); } diff --git a/shared/src/model/auth/role.rs b/shared/src/model/auth/role.rs index 22f8ccad8..c8459eb68 100644 --- a/shared/src/model/auth/role.rs +++ b/shared/src/model/auth/role.rs @@ -120,7 +120,7 @@ mod tests { let set = RoleSet::new(); assert!(set.is_empty()); assert!(!set.contains(Role::Admin)); - assert!(set.names().is_empty()); + assert_eq!(set.names().len(), 0); } #[test] diff --git a/shared/src/model/config/curation.rs b/shared/src/model/config/curation.rs index dcada51b0..227b0b689 100644 --- a/shared/src/model/config/curation.rs +++ b/shared/src/model/config/curation.rs @@ -195,7 +195,7 @@ mod tests { dto.prepare(&xtream()).expect("missing token is not a config-load failure"); assert!(dto.trakt.is_none()); let tmdb = dto.tmdb.unwrap(); - assert!(tmdb.api.access_token.is_empty()); + assert_eq!(tmdb.api.access_token.len(), 0); assert!(tmdb.enabled); assert_eq!(tmdb.trending[0].kind, TmdbTrendingKind::Movie); assert_eq!(tmdb.trending[0].time_window, TmdbTrendingTimeWindow::Week); diff --git a/shared/src/model/config/epg/tests.rs b/shared/src/model/config/epg/tests.rs index 010632123..15abe1a9b 100644 --- a/shared/src/model/config/epg/tests.rs +++ b/shared/src/model/config/epg/tests.rs @@ -52,7 +52,7 @@ fn include_computed_false_skips_resolution() { let mut cfg = EpgConfigDto { sources: Some(vec![xmltv_source("provider://myprovider/xmltv.php")]), ..Default::default() }; cfg.prepare(|| Err("no auto".to_owned()), false).expect("prepare with include_computed=false should succeed"); - assert!(cfg.t_sources.is_empty()); + assert_eq!(cfg.t_sources.len(), 0); } #[test] diff --git a/shared/src/model/config/target/tests.rs b/shared/src/model/config/target/tests.rs index 5b997d299..5cf38f5ad 100644 --- a/shared/src/model/config/target/tests.rs +++ b/shared/src/model/config/target/tests.rs @@ -38,7 +38,7 @@ fn shipped_tmdb_example_is_safe_by_default_and_prepares_when_explicitly_enabled( .expect("shipped TMDB example"); assert!(!target.enabled); assert!(!target.curation.as_ref().unwrap().enabled); - assert!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.is_empty()); + assert_eq!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.len(), 0); target.prepare(1, None, None).unwrap(); target.enabled = true; target.curation.as_mut().unwrap().enabled = true; diff --git a/shared/src/model/config/trakt.rs b/shared/src/model/config/trakt.rs index 33a47d737..288247d93 100644 --- a/shared/src/model/config/trakt.rs +++ b/shared/src/model/config/trakt.rs @@ -311,7 +311,7 @@ mod tests { assert_eq!(config.catalog_selection, TraktCatalogSelection::Full); assert!(config.include_xtream_base_categories); - assert!(config.lists.is_empty()); + assert_eq!(config.lists.len(), 0); assert_eq!(config.charts.len(), 1); assert_eq!(config.charts[0].kind, TraktChartKind::Movies); assert_eq!(config.charts[0].kind.content_type(), TraktContentType::Vod); diff --git a/shared/src/model/playlist_update_status.rs b/shared/src/model/playlist_update_status.rs index a1c5583a5..b71620b2c 100644 --- a/shared/src/model/playlist_update_status.rs +++ b/shared/src/model/playlist_update_status.rs @@ -188,8 +188,8 @@ mod tests { let decoded = serde_json::from_str::(r#"{"inputs":[{"input_id":7,"last_update":1725000000}]}"#)?; - assert!(decoded.inputs[0].clusters.is_empty()); - assert!(decoded.active_updates.is_empty()); + assert_eq!(decoded.inputs[0].clusters.len(), 0); + assert_eq!(decoded.active_updates.len(), 0); Ok(()) } diff --git a/shared/src/model/stalker.rs b/shared/src/model/stalker.rs index 42d9ebddf..6c19e0328 100644 --- a/shared/src/model/stalker.rs +++ b/shared/src/model/stalker.rs @@ -495,7 +495,7 @@ mod tests { assert!(!descriptor.is_empty()); descriptor.clean(); // clean() drops empty candidates, so the descriptor is now fully empty. - assert!(descriptor.candidates.is_empty()); + assert_eq!(descriptor.candidates.len(), 0); assert!(descriptor.capabilities.is_none()); assert!(descriptor.is_empty()); } diff --git a/shared/src/utils/hash_utils/tests/hashing_ids.rs b/shared/src/utils/hash_utils/tests/hashing_ids.rs index a02479b4e..570e79a37 100644 --- a/shared/src/utils/hash_utils/tests/hashing_ids.rs +++ b/shared/src/utils/hash_utils/tests/hashing_ids.rs @@ -34,7 +34,7 @@ fn numeric_id_with_trailing_slash() { fn non_numeric_last_segment_returns_hash() { let result = extract_id_from_url("http://server.com/live/user/pass/hello.ts"); assert!(!result.chars().all(|c| c.is_ascii_digit()), "should be a hash, not a numeric id"); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] @@ -52,13 +52,13 @@ fn no_slash_returns_hash() { #[test] fn empty_url_returns_hash() { let result = extract_id_from_url(""); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] fn only_slashes_returns_hash() { let result = extract_id_from_url("///"); - assert!(!result.is_empty()); + assert_ne!(result.len(), 0); } #[test] diff --git a/test/fixtures/hls/reduced-retry/baseline.m3u8 b/test/fixtures/hls/reduced-retry/baseline.m3u8 new file mode 100644 index 000000000..30da14cc1 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/baseline.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1000 +#EXTINF:4.0, +a1000.ts +#EXTINF:4.0, +a1001.ts +#EXTINF:4.0, +a1002.ts +#EXTINF:4.0, +a1003.ts +#EXTINF:4.0, +a1004.ts +#EXTINF:4.0, +a1005.ts diff --git a/test/fixtures/hls/reduced-retry/conflict.m3u8 b/test/fixtures/hls/reduced-retry/conflict.m3u8 new file mode 100644 index 000000000..0418d87c8 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/conflict.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1003 +#EXTINF:4.0, +b1003.ts +#EXTINF:4.0, +b1004.ts +#EXTINF:4.0, +b1005.ts +#EXTINF:4.0, +b1006.ts +#EXTINF:4.0, +b1007.ts +#EXTINF:4.0, +b1008.ts diff --git a/test/fixtures/hls/reduced-retry/repaired.m3u8 b/test/fixtures/hls/reduced-retry/repaired.m3u8 new file mode 100644 index 000000000..1e304d530 --- /dev/null +++ b/test/fixtures/hls/reduced-retry/repaired.m3u8 @@ -0,0 +1,16 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:1003 +#EXTINF:4.0, +a1003.ts +#EXTINF:4.0, +a1004.ts +#EXTINF:4.0, +a1005.ts +#EXTINF:4.0, +a1006.ts +#EXTINF:4.0, +a1007.ts +#EXTINF:4.0, +a1008.ts