diff --git a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs index f1eb75302..06f069fe6 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs @@ -87,6 +87,7 @@ pub(in crate::api::endpoints::hls_api) async fn prepare_hls_canonical_manifest_o let strip = app_state.hls.proxy.strip(); match hls_provider_connections_exhausted_manifest_resolution( app_state, + Some(fingerprint), session, &context.username, origin.input, @@ -385,6 +386,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest join_hls_canonical_manifest_owner( HlsCanonicalOwnerHandoffContext { app_state, + fingerprint: Some(fingerprint), proxy_session_id: path_proxy_session_id, access_lease_id, expected_lease_issued_at_ms, @@ -431,6 +433,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest let strip = app_state.hls.proxy.strip(); if let Some(response) = try_hls_cached_manifest_response( app_state, + Some(fingerprint), &session, access_lease_id, access_lease_state, @@ -472,6 +475,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest let strip = app_state.hls.proxy.strip(); if let Some(response) = try_hls_cached_manifest_response( app_state, + Some(fingerprint), &session, access_lease_id, access_lease_state, @@ -502,6 +506,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest let strip = app_state.hls.proxy.strip(); if let Some(live_response) = try_hls_cached_manifest_response( app_state, + Some(fingerprint), &session, access_lease_id, access_lease_state, @@ -535,6 +540,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest let strip = app_state.hls.proxy.strip(); if let Some(response) = try_hls_cached_manifest_response( app_state, + Some(fingerprint), &session, access_lease_id, access_lease_state, diff --git a/backend/app/src/api/endpoints/hls_api/manifest/cached.rs b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs index 0b7e2a3dc..79366b04b 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/cached.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs @@ -26,6 +26,108 @@ use tuliprox_hls::api::{ ProxySessionId, }; +#[allow(clippy::too_many_arguments)] +async fn prepare_live_window_response( + app_state: &Arc, + session: &HlsSessionHandle, + view: &HlsCachedManifestViewContext<'_>, + fingerprint: Option<&crate::model::Fingerprint>, + options: HlsCachedManifestOptions, + started_at_ms: u64, +) -> Option<(HlsMaterializedSharedManifest, HlsLeaseManifestSnapshot, HlsPublishedTransientResourceIds, u64)> { + let now_ms = current_time_millis(); + let lease = + app_state.hls.proxy.access_lease_response_snapshot(view.access_lease_id, view.proxy_session_id, now_ms).await?; + let (prepared, queued) = { + let mut state = session.write().await; + state.live_window_strip = app_state.hls.proxy.strip(); + if !tuliprox_hls::api::hls_live_window_origin_allowed_for_request(&state, options, started_at_ms, now_ms) { + return None; + } + // Keep ordinary forward session commits current when their own window + // is ready. A lease's independent entry never depends on this attempt. + let _ = state.render_and_store_manifest(now_ms); + let desired = state.plan_live_window(tuliprox_hls::api::HlsLiveWindowView::for_lease(&lease))?; + if let Some(previous) = lease + .last_manifest_snapshot + .as_ref() + .filter(|previous| desired.visible.first().is_some_and(|first| *first < previous.first_proxy_seq)) + { + if now_ms > previous.delivered_at_ms.saturating_add(previous.playlist_duration_ms) { + return None; + } + let published = previous.startup_revisions.as_ref()?.window_plan.as_ref()?; + return Some(( + HlsMaterializedSharedManifest { + body: published.body.to_string(), + mode: "normal", + initial_strip_outcome: None, + }, + previous.clone(), + HlsPublishedTransientResourceIds::default(), + now_ms, + )); + } + let plan = state.plan_live_window_for_lease(&lease)?; + let backpressure = app_state.hls.proxy.segment_worker_pool().classify_backpressure_for_session(&state); + let report = state.queue_manifest_fetch_candidates(now_ms, backpressure.allows_prefetch()); + let queued = report.render_window_queued + report.prefetch_queued > 0; + let prepared = tuliprox_hls::api::HlsManifestRenderer::render_live_window(&state, &plan, now_ms).ok().and_then( + |rendered| { + // An explicit planned view bypasses generic strip. PreserveFullManifest + // continues to mean the origin's protected manifest semantics only. + let materialized = HlsMaterializedSharedManifest { + body: super::materialize_hls_access_manifest( + &rendered.body, + view.access_lease_id, + view.server_path, + ), + mode: "normal", + initial_strip_outcome: hls_access_manifest_uses_startup_view(lease.state).then_some( + tuliprox_hls::initial_strip::HlsInitialStripOutcome::Applied { + mode: match view.strip.mode { + shared::model::HlsStripMode::Segments => "segments", + shared::model::HlsStripMode::Seconds => "seconds", + }, + configured: view.strip.value, + effective: plan.effective_strip_segments, + visible_segments: plan.visible.len(), + }, + ), + }; + let snapshot = observe_hls_lease_manifest_snapshot_derivation( + app_state, + view.proxy_session_id, + view.access_lease_id, + derive_hls_lease_manifest_snapshot( + &HlsLeaseManifestSnapshotInput::FastStartWindow { + session: &state, + plan: &plan, + rendered: &rendered, + materialized_body: &materialized.body, + }, + now_ms, + ), + ) + .ok()??; + Some((materialized, snapshot, HlsPublishedTransientResourceIds::default(), now_ms)) + }, + ); + (prepared, queued) + }; + if queued { + if let Some(fingerprint) = fingerprint { + let context = + super::super::build_hls_segment_fetch_context(app_state, session, None, fingerprint, None).await; + let worker = Arc::clone(app_state.hls.proxy.segment_worker_pool()); + tokio::spawn(async move { + worker.wake_scheduler(context, current_time_millis()).await; + }); + } + } + prepared +} + pub(in crate::api::endpoints::hls_api) struct HlsCachedManifestRead { pub(in crate::api::endpoints::hls_api) transient_body: Option, pub(in crate::api::endpoints::hls_api) rendered_body: Option, @@ -246,6 +348,7 @@ pub(in crate::api::endpoints::hls_api) fn observe_hls_lease_manifest_snapshot_de #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response( app_state: &Arc, + fingerprint: Option<&crate::model::Fingerprint>, session: &HlsSessionHandle, access_lease_id: &HlsAccessLeaseId, access_lease_state: HlsAccessLeaseState, @@ -273,10 +376,27 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response server_path, bandwidth_learning, ); + let mut last_wait_reason = None; loop { let cached = read_hls_cached_manifest(session, options, started_at_ms).await; - if !cached.wait_for_initial_commit { - let prepared = if let Some(transient) = cached.transient_body { + let fast_start = session.read().await.uses_fast_start_live_window(); + if cached.wait_for_initial_commit || (cached.rendered_body.is_none() && cached.transient_body.is_none()) { + log_hls_manifest_wait_transition(&mut last_wait_reason, "fresh_render_required", session, access_lease_id) + .await; + } + if fast_start || !cached.wait_for_initial_commit { + let prepared = if fast_start { + let prepared = + prepare_live_window_response(app_state, session, &view, fingerprint, options, started_at_ms).await; + if prepared.is_none() { + if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { + continue; + } + return (access_lease_state != HlsAccessLeaseState::Pending) + .then(hls_cached_manifest_temporarily_unavailable); + } + prepared + } else if let Some(transient) = cached.transient_body { let materialized = view.materialize(&transient.body, "transient", transient.window_policy); let published_resource_ids = if transient.window_policy.preserves_full_manifest() { transient.published_resource_ids.clone() @@ -342,6 +462,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response if access_lease_state != HlsAccessLeaseState::Pending { return None; } + log_hls_manifest_wait_transition( + &mut last_wait_reason, + "missing_revision_or_head_readiness", + session, + access_lease_id, + ) + .await; if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { continue; } @@ -356,6 +483,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response && !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms) .await { + log_hls_manifest_wait_transition( + &mut last_wait_reason, + "admission_rejected", + session, + access_lease_id, + ) + .await; if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { continue; } @@ -375,6 +509,12 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response admission_at_ms, ) .await; + if fast_start && !outcome.is_committed() { + if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { + continue; + } + return Some(hls_cached_manifest_temporarily_unavailable()); + } if let Some(snapshot_generation) = outcome.snapshot_generation() { let published_at_ms = current_time_millis(); let first_startup_publication = @@ -426,3 +566,28 @@ pub(in crate::api::endpoints::hls_api) async fn wait_for_hls_startup_evidence( tokio::time::sleep(wait_timeout.saturating_sub(elapsed).min(Duration::from_millis(25))).await; true } + +/// Per-request state prevents a 25-ms polling loop from repeating diagnostics. +async fn log_hls_manifest_wait_transition( + previous: &mut Option<(&'static str, bool, bool)>, + reason: &'static str, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, +) { + let session = session.read().await; + let degraded = session.origin_control.path_condition.is_degraded(); + let head = session.publishable_origin_head_proxy_seq.and_then(|seq| session.startup.as_ref()?.revisions.get(&seq)); + let head_ready = head.is_some_and(|head| head.revision().is_startup_head_ready(session.startup_mode())); + let state = (reason, degraded, head_ready); + if *previous == Some(state) { + return; + } + *previous = Some(state); + debug!( + "HLS playlist wait: lease={} proxy_session={} reason={reason} degraded={degraded} head_ready={head_ready} origin_generation={} readiness_generation={} head_revision={:?} head_prefix={}", + safe_hls_access_lease_id(lease_id), safe_proxy_session_id(&session.proxy_session_id), + session.activity.origin_work_generation, session.activity.media_readiness_generation, + head.map(|head| head.revision().id()), + head.map_or(0, |head| head.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire)), + ); +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs index 4232efed7..bd773e580 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs @@ -166,6 +166,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_canonical_owner_failed( pub(in crate::api::endpoints::hls_api) struct HlsCanonicalOwnerHandoffContext<'a> { pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) fingerprint: Option<&'a crate::model::Fingerprint>, pub(in crate::api::endpoints::hls_api) proxy_session_id: &'a ProxySessionId, pub(in crate::api::endpoints::hls_api) access_lease_id: &'a HlsAccessLeaseId, pub(in crate::api::endpoints::hls_api) expected_lease_issued_at_ms: Option, @@ -248,6 +249,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han ); if let Some(response) = try_hls_cached_manifest_response( context.app_state, + context.fingerprint, current_session, context.access_lease_id, lease.state, diff --git a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs index d8fc1da7b..ce5ec8171 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs @@ -622,6 +622,7 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline_manifest_response( app_state: &Arc, + fingerprint: Option<&Fingerprint>, session: &HlsSessionHandle, access_lease_id: &HlsAccessLeaseId, access_lease_state: HlsAccessLeaseState, @@ -668,6 +669,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline try_hls_cached_manifest_response( app_state, + fingerprint, session, access_lease_id, access_lease_state, @@ -682,6 +684,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline #[allow(clippy::too_many_arguments)] pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_or_provider_exhausted_response( app_state: &Arc, + fingerprint: Option<&Fingerprint>, session: &HlsSessionHandle, username: &str, input: &ConfigInput, @@ -715,6 +718,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_or_provi } if let Some(response) = hls_shared_provisioning_timeline_manifest_response( app_state, + fingerprint, session, access_lease_id, access_lease_state, @@ -757,6 +761,7 @@ pub(in crate::api::endpoints::hls_api) enum HlsProviderExhaustedResolution { #[allow(clippy::too_many_arguments)] pub(in crate::api::endpoints::hls_api) async fn hls_provider_connections_exhausted_manifest_resolution( app_state: &Arc, + fingerprint: Option<&Fingerprint>, session: &HlsSessionHandle, username: &str, input: &ConfigInput, @@ -793,6 +798,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_provider_connections_exhaust HlsProviderExhaustedResolution::Response( hls_shared_provisioning_or_provider_exhausted_response( app_state, + fingerprint, session, username, input, diff --git a/backend/app/src/api/endpoints/hls_api/segment/segments.rs b/backend/app/src/api/endpoints/hls_api/segment/segments.rs index f80c72b60..e8fd2309a 100644 --- a/backend/app/src/api/endpoints/hls_api/segment/segments.rs +++ b/backend/app/src/api/endpoints/hls_api/segment/segments.rs @@ -29,6 +29,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment( fingerprint: Fingerprint, axum::extract::Path(params): axum::extract::Path, axum::extract::State(app_state): axum::extract::State>, + method: Method, headers: HeaderMap, ) -> axum::response::Response { let proxy_session_id = ProxySessionId(params.proxy_session_id); @@ -83,6 +84,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment( &access_context, &fingerprint, &headers, + &method, segment_file, now_ms, ) @@ -162,12 +164,17 @@ pub(in crate::api::endpoints::hls_api) async fn demand_fetch_hls_live_segment( } } +#[allow( + clippy::too_many_arguments, + reason = "Retains request method and headers alongside the validated lease context." +)] pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_lease( app_state: &Arc, session: &HlsSessionHandle, access_context: &HlsAccessContext, fingerprint: &Fingerprint, headers: &HeaderMap, + method: &Method, segment_file: HlsSegmentFile, now_ms: u64, ) -> axum::response::Response { @@ -181,7 +188,9 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_le let Some(lease_identity) = current_lease.media_identity() else { return StatusCode::NOT_FOUND.into_response(); }; - let response_context = hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await; + let mut response_context = + hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await; + response_context.request_method = method.clone(); let response = hls_resource_serve_outcome_response( app_state, access_context, diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index 092208599..3ab01936f 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -91,6 +91,8 @@ mod headers; mod late_custom_response; mod lease_lifecycle; mod manifest_entry_freshness; +mod manifest_live_window; +mod manifest_revision_recovery; mod manifest_short_window; mod manifest_startup; mod observability; diff --git a/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs b/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs index 5883cfa25..d72168ed7 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs @@ -154,6 +154,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_runtime_bandwidth_man let response = super::super::try_hls_cached_manifest_response( &app_state, + None, &session, &access_lease_id, HlsAccessLeaseState::Pending, diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_live_window.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_live_window.rs new file mode 100644 index 000000000..b583118dd --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_live_window.rs @@ -0,0 +1,583 @@ +use super::{normal_manifest, response_body, test_app_state_with_hls_proxy, try_test_hls_cached_manifest_response}; +use crate::api::model::{HlsAccessLeaseId, HlsAccessLeaseState, HlsProxyManager, HlsSessionKey, SegmentCacheStatus}; +use axum::http::StatusCode; +use shared::model::{HlsCacheConfigDto, HlsStartupMode, HlsStripMode}; +use std::{fmt::Write as _, sync::Arc, time::Duration}; +use tuliprox_core::{ + model::{HlsCacheConfig, StripConfig}, + utils::current_time_millis, +}; + +fn origin( + first: u64, + count: usize, + durations: &[u64], +) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-MEDIA-SEQUENCE:{first}\n"); + for index in 0..count { + let sequence = first + u64::try_from(index).unwrap(); + writeln!(body, "#EXTINF:{},\nhttp://origin.invalid/{sequence}.ts", durations[index % durations.len()]).unwrap(); + } + normal_manifest(&body) +} + +fn prepare_revisions(session: &mut tuliprox_hls::api::HlsSession, sequences: &[u64]) { + let mode = session.startup_mode(); + let startup = session.startup.as_mut().unwrap(); + for seq in sequences { + if startup.revisions.contains_key(seq) { + continue; + } + let revision = startup + .store + .create( + session.proxy_session_id.clone(), + *seq, + if mode == HlsStartupMode::FirstReady { + tuliprox_hls::SegmentRevisionKind::Processed + } else { + tuliprox_hls::SegmentRevisionKind::Raw + }, + ) + .unwrap(); + revision + .revision() + .complete(tuliprox_hls::api::CachedSegmentMetadata { path: std::path::PathBuf::from("unused"), size: 1 }); + startup.revisions.insert(*seq, revision); + session.segments.get_mut(seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: current_time_millis() }; + } +} + +#[tokio::test] +async fn live_window_timeout_preserves_unpublished_fallback_and_existing_lease_retries() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + super::enable_channel_unavailable_custom_response(&app); + let session = manager + .get_or_create_session(HlsSessionKey::new(1, "unready-window"), b"secret", current_time_millis()) + .await; + let proxy_id = session.read().await.proxy_session_id.clone(); + for has_origin in [false, true] { + if has_origin { + session.write().await.apply_origin_manifest(&origin(0, 6, &[4])).unwrap(); + } + for state in [HlsAccessLeaseState::Pending, HlsAccessLeaseState::Activated, HlsAccessLeaseState::Idle] { + for wait in [Duration::ZERO, Duration::from_millis(1)] { + let id = HlsAccessLeaseId(format!("timeout-{has_origin}-{state:?}-{wait:?}")); + super::prepare_pending_test_hls_access_lease(&app, &proxy_id, &id).await; + let mut lease = + manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap(); + lease.state = state; + if state == HlsAccessLeaseState::Activated { + lease.active_until_ms = Some(current_time_millis() + 60_000); + } + manager.prepare_access_lease(lease).await; + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &id, + state, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + super::HlsCachedManifestOptions::initial(wait), + ) + .await; + if state == HlsAccessLeaseState::Pending { + assert!(response.is_none(), "{mode}: unpublished fallback must remain reachable"); + let fallback = + super::super::hls_unpublished_lease_channel_unavailable_response(&app, &proxy_id, &id) + .await; + assert_eq!(fallback.status(), StatusCode::OK); + let body = String::from_utf8(response_body(fallback).await.to_vec()).unwrap(); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(!body.contains("/hls/shared/live/")); + super::disable_custom_stream_response(&app); + let fallback = + super::super::hls_unpublished_lease_channel_unavailable_response(&app, &proxy_id, &id) + .await; + assert_eq!(fallback.status(), StatusCode::NOT_FOUND); + super::enable_channel_unavailable_custom_response(&app); + } else { + let response = response.expect("existing leases keep the temporary failure response"); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(axum::http::header::RETRY_AFTER)); + } + let lease = + manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap(); + assert!(lease.last_manifest_snapshot.is_none()); + } + } + } + } +} + +#[tokio::test] +async fn live_window_provisioning_fallback_wakes_scheduler_for_the_requesting_lease() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + max_segments_prefetch: 0, + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.value = 1; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + super::owner_handoff::enable_hls_provisioning_custom_response(&app); + let session = manager + .get_or_create_session(HlsSessionKey::new(1, "provisioning-window"), b"secret", current_time_millis()) + .await; + { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + prepare_revisions(&mut state, &[3, 4, 5]); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let proxy_id = session.read().await.proxy_session_id.clone(); + let id = HlsAccessLeaseId("provisioning-lease".into()); + super::prepare_pending_test_hls_access_lease(&app, &proxy_id, &id).await; + // Pause before origin I/O: this test verifies the scheduler handoff, + // independently of whether a provider slot is available afterwards. + let gate = manager.segment_worker_pool().pause_next_origin_prepare_for_test(); + let response = super::super::hls_shared_provisioning_timeline_manifest_response( + &app, + Some(&super::test_fingerprint()), + &session, + &id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 1 }, + None, + ) + .await; + assert!(response.is_none(), "the requesting lease still needs its earlier entry segment"); + tokio::time::timeout(Duration::from_secs(1), gate.entered.notified()) + .await + .expect("the fallback must wake the scheduler for queued live-window work"); + { + let state = session.read().await; + assert_eq!(state.segment_prefetch_queue.proxy_seqs(), vec![2]); + assert!(matches!( + state.segments[&2].status, + SegmentCacheStatus::Queued { priority: tuliprox_hls::api::SegmentFetchPriority::RenderWindow, .. } + )); + } + assert!(manager.access_leases().write().await.remove_access_lease(&id).is_some()); + gate.prepare.notify_one(); + tokio::time::timeout(Duration::from_secs(1), gate.prepared.notified()).await.unwrap(); + gate.proceed.notify_one(); + } +} + +#[tokio::test] +async fn live_window_api_matrix_has_exact_playlist_snapshot_and_no_second_strip() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + for (count, strip_mode, strip, expected) in [ + (6, HlsStripMode::Segments, 0, vec![3, 4, 5]), + (6, HlsStripMode::Segments, 1, vec![2, 3, 4]), + (6, HlsStripMode::Segments, 2, vec![1, 2, 3]), + (6, HlsStripMode::Segments, 3, vec![0, 1, 2]), + (6, HlsStripMode::Segments, u64::MAX, vec![0, 1, 2]), + (12, HlsStripMode::Segments, 1, vec![8, 9, 10]), + (6, HlsStripMode::Seconds, 0, vec![3, 4, 5]), + (6, HlsStripMode::Seconds, 7, vec![2, 3, 4]), + (6, HlsStripMode::Seconds, 8, vec![1, 2, 3]), + (6, HlsStripMode::Seconds, 10, vec![1, 2, 3]), + (6, HlsStripMode::Seconds, u64::MAX, vec![0, 1, 2]), + ] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.mode = strip_mode; + dto.strip.value = strip; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = + manager.get_or_create_session(HlsSessionKey::new(1, "window"), b"secret", current_time_millis()).await; + { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(9_000_000, count, &[4, 4, 4, 4, 3, 7])).unwrap(); + state.live_window_strip = StripConfig { mode: strip_mode, value: strip }; + prepare_revisions(&mut state, &expected); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let id = HlsAccessLeaseId("entry".into()); + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: strip_mode, value: strip }, + Some("/prefix"), + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{mode} {strip_mode:?} {strip}"); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + let lease = manager + .access_lease_response_snapshot(&id, &session.read().await.proxy_session_id, current_time_millis()) + .await + .unwrap(); + let snapshot = lease.last_manifest_snapshot.unwrap(); + assert_eq!(snapshot.visible_proxy_seqs().collect::>(), expected); + let uris = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::>(); + assert_eq!(uris.len(), 3); + for (uri, seq) in uris.iter().zip(expected) { + assert!(uri.starts_with("/prefix/hls/shared/live/")); + assert!(uri.ends_with(&format!("/{seq:06}.ts"))); + assert!(uri.contains("/entry/")); + } + } + } +} + +#[tokio::test] +async fn live_window_entry_does_not_depend_on_the_saved_shared_manifest_prefix() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.value = 1; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = manager + .get_or_create_session(HlsSessionKey::new(1, "saved-prefix"), b"secret", current_time_millis()) + .await; + let saved = { + let mut state = session.write().await; + let startup = state.startup.take(); + state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap(); + for entry in state.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: current_time_millis() }; + } + let saved = state.render_and_store_manifest(current_time_millis()).unwrap(); + assert_eq!(saved.segment_proxy_seqs, vec![0, 1, 2, 3, 4, 5]); + state.startup = startup; + for entry in state.segments.values_mut() { + entry.status = SegmentCacheStatus::Discovered; + } + prepare_revisions(&mut state, &[2, 3, 4]); + saved.body + }; + let id = HlsAccessLeaseId("new-lease".into()); + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 1 }, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:2\n")); + assert_eq!(body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()).count(), 3); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().unwrap().body, saved); + } +} + +#[tokio::test] +async fn live_window_leases_enter_independently_and_active_windows_only_grow_forward() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.value = 1; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = + manager.get_or_create_session(HlsSessionKey::new(1, "leases"), b"secret", current_time_millis()).await; + { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + prepare_revisions(&mut state, &[2, 3, 4]); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + let first = HlsAccessLeaseId("first".into()); + for (slide, expected) in [ + (None, vec![2, 3, 4]), + (Some(0), vec![2, 3, 4, 5]), + (Some(1), vec![2, 3, 4, 5, 6]), + (Some(2), vec![2, 3, 4, 5, 6, 7]), + (Some(2), vec![2, 3, 4, 5, 6, 7]), + ] { + if let Some(slide) = slide { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(slide, 6, &[4])).unwrap(); + prepare_revisions(&mut state, &expected); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &first, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let _ = response_body(response).await; + let id = session.read().await.proxy_session_id.clone(); + let mut lease = manager.access_lease_response_snapshot(&first, &id, current_time_millis()).await.unwrap(); + assert_eq!( + lease.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::>(), + expected + ); + // Preserve this exact lease's bindings and progressive claim state. + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(current_time_millis() + 60_000); + manager.prepare_access_lease(lease).await; + } + let second = HlsAccessLeaseId("second".into()); + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &second, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let _ = response_body(response).await; + let id = session.read().await.proxy_session_id.clone(); + let lease = manager.access_lease_response_snapshot(&second, &id, current_time_millis()).await.unwrap(); + assert_eq!(lease.last_manifest_snapshot.unwrap().visible_proxy_seqs().collect::>(), vec![4, 5, 6]); + let first = manager.access_lease_response_snapshot(&first, &id, current_time_millis()).await.unwrap(); + assert_eq!( + first.last_manifest_snapshot.unwrap().visible_proxy_seqs().collect::>(), + vec![2, 3, 4, 5, 6, 7] + ); + } +} + +#[tokio::test] +async fn live_window_active_slides_preserve_duration_without_padding_another_leases_entry() { + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.value = 0; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = manager + .get_or_create_session(HlsSessionKey::new(1, "duration-slides"), b"secret", current_time_millis()) + .await; + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + let id = HlsAccessLeaseId("active".into()); + for (first, durations, expected) in [ + (100, vec![6, 6, 6, 5], vec![1, 2, 3]), + (101, vec![6, 6, 5, 7], vec![1, 2, 3, 4]), + (103, vec![5, 7, 6], vec![2, 3, 4, 5]), + ] { + { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(first, durations.len(), &durations)).unwrap(); + prepare_revisions(&mut state, &expected); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{mode} slide {first}"); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", expected[0]))); + assert_eq!(body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()).count(), expected.len()); + let proxy_id = session.read().await.proxy_session_id.clone(); + let mut lease = + manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap(); + let snapshot = lease.last_manifest_snapshot.as_ref().unwrap(); + assert_eq!(snapshot.visible_proxy_seqs().collect::>(), expected); + if first != 100 { + assert!(snapshot.playlist_duration_ms >= 21_000); + } + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(current_time_millis() + 60_000); + manager.prepare_access_lease(lease).await; + } + let new_id = HlsAccessLeaseId("new".into()); + // Both the first publication and its active refresh must honor this lease's own head. + for _ in 0..2 { + let response = try_test_hls_cached_manifest_response( + &app, + &session, + &new_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let _ = response_body(response).await; + let proxy_id = session.read().await.proxy_session_id.clone(); + let mut lease = + manager.access_lease_response_snapshot(&new_id, &proxy_id, current_time_millis()).await.unwrap(); + let snapshot = lease.last_manifest_snapshot.as_ref().unwrap(); + assert_eq!(snapshot.visible_proxy_seqs().collect::>(), vec![3, 4, 5]); + assert_eq!(snapshot.playlist_duration_ms, 18_000); + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(current_time_millis() + 60_000); + manager.prepare_access_lease(lease).await; + } + } +} + +#[tokio::test] +#[expect(clippy::too_many_lines, reason = "One ordered reload, re-entry and stale-publication transaction.")] +async fn live_window_config_changes_reentry_and_generation_races_preserve_lease_evidence() { + use tuliprox_hls::api::{ + derive_hls_lease_manifest_snapshot, HlsLeaseManifestSnapshotInput, HlsLiveWindowView, HlsManifestRenderer, + }; + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + ..Default::default() + }; + dto.startup.mode = mode; + dto.strip.value = 1; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = + manager.get_or_create_session(HlsSessionKey::new(1, "races"), b"secret", current_time_millis()).await; + { + let mut state = session.write().await; + state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + prepare_revisions(&mut state, &[2, 3, 4, 5]); + state.render_and_store_manifest(current_time_millis()).unwrap(); + } + let id = HlsAccessLeaseId("entry".into()); + let request_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + let request = || { + try_test_hls_cached_manifest_response( + &app, + &session, + &id, + HlsAccessLeaseState::Pending, + &request_strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + }; + let first = request().await.unwrap(); + assert_eq!(first.status(), StatusCode::OK); + let first_body = response_body(first).await; + let proxy = session.read().await.proxy_session_id.clone(); + let original = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap(); + let mut leases = manager.access_leases().write().await; + if mode != HlsStartupMode::FirstReady { + assert!(leases.try_claim_progressive_startup(&id, 2, current_time_millis())); + } + drop(leases); + // A larger strip must retain the published body, never rewind to 0,1,2. + dto.strip.value = 3; + let mut config = app.app_config.config.load().as_ref().clone(); + config.reverse_proxy = + Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto { + hls_cache: Some(dto.clone()), + ..Default::default() + })); + app.app_config.config.store(Arc::new(config)); + manager.update_config(&app.app_config).await; + let retained = request().await.unwrap(); + assert_eq!(retained.status(), StatusCode::OK); + assert_eq!(response_body(retained).await, first_body); + dto.strip.value = 0; + let mut config = app.app_config.config.load().as_ref().clone(); + config.reverse_proxy.as_mut().unwrap().hls_cache = Some(HlsCacheConfig::from(&dto)); + app.app_config.config.store(Arc::new(config)); + manager.update_config(&app.app_config).await; + let next = request().await.unwrap(); + assert_eq!(next.status(), StatusCode::OK); + let _ = response_body(next).await; + let mut lease = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap(); + assert_eq!( + lease.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::>(), + vec![3, 4, 5] + ); + lease.state = HlsAccessLeaseState::Idle; + manager.prepare_access_lease(lease).await; + let idle = request().await.unwrap(); + assert_eq!(idle.status(), StatusCode::OK); + let _ = response_body(idle).await; + if mode != HlsStartupMode::FirstReady { + assert!(!manager.access_leases().write().await.try_claim_progressive_startup( + &id, + 3, + current_time_millis() + )); + } + let guard = + manager.prepare_access_lease_manifest_publication(&id, &proxy, current_time_millis()).await.unwrap(); + let stale = { + let state = session.read().await; + let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap(); + let rendered = HlsManifestRenderer::render_live_window(&state, &plan, current_time_millis()).unwrap(); + derive_hls_lease_manifest_snapshot( + &HlsLeaseManifestSnapshotInput::FastStartWindow { + session: &state, + plan: &plan, + rendered: &rendered, + materialized_body: &rendered.body, + }, + current_time_millis(), + ) + .unwrap() + .unwrap() + }; + session.write().await.activity.origin_work_generation += 1; + assert!(!manager + .commit_access_lease_manifest_publication(&id, &proxy, guard, stale, current_time_millis()) + .await + .is_committed()); + let after = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap(); + assert_eq!( + after.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::>(), + vec![3, 4, 5] + ); + assert_eq!(after.issued_at_ms, original.issued_at_ms); + } +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_revision_recovery.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_revision_recovery.rs new file mode 100644 index 000000000..5916b95b1 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_revision_recovery.rs @@ -0,0 +1,191 @@ +use super::{normal_manifest, response_body, test_app_state_with_hls_proxy, try_test_hls_cached_manifest_response}; +use crate::api::model::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, + SegmentCacheStatus, +}; +use axum::http::{HeaderMap, StatusCode}; +use shared::model::{HlsCacheConfigDto, HlsStartupMode}; +use std::{fmt::Write as _, sync::Arc, time::Duration}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tuliprox_core::{ + model::{HlsCacheConfig, StripConfig}, + utils::current_time_millis, +}; +use tuliprox_hls::api::SegmentFetchContext; + +#[tokio::test] +#[allow(clippy::too_many_lines, reason = "One deterministic held-body scheduling and HTTP publication scenario.")] +async fn retained_playlist_request_is_released_by_successor_headers_while_bodies_stay_open() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("origin"); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let (headers, header_gate) = tokio::sync::watch::channel(false); + let (bodies, body_gate) = tokio::sync::watch::channel(false); + let origin_task = tokio::spawn(async move { + let mut children = tokio::task::JoinSet::new(); + loop { + tokio::select! { + socket = listener.accept() => { + let (socket, _) = socket.unwrap(); + let mut headers = header_gate.clone(); + let mut bodies = body_gate.clone(); + children.spawn(async move { + let mut socket = BufReader::new(socket); + let mut request = String::new(); + socket.read_line(&mut request).await.unwrap(); + let head = request.contains("/103.ts "); + loop { + let mut line = String::new(); + socket.read_line(&mut line).await.unwrap(); + if line == "\r\n" || line.is_empty() { break; } + } + let mut socket = socket.into_inner(); + if !head { + while !*headers.borrow_and_update() { headers.changed().await.unwrap(); } + } + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\n").await.unwrap(); + if head { socket.write_all(b"segment-").await.unwrap(); } + while !*bodies.borrow_and_update() { bodies.changed().await.unwrap(); } + socket.write_all(if head { b"body".as_slice() } else { b"segment-body".as_slice() }).await.unwrap(); + }); + } + _ = children.join_next(), if !children.is_empty() => {} + } + } + }); + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + max_concurrent_segment_fetches_per_session: 3, + ..Default::default() + }; + dto.startup.mode = HlsStartupMode::Optimistic; + let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto))); + let app = test_app_state_with_hls_proxy(manager.clone()); + let session = + manager.get_or_create_session(HlsSessionKey::new(1, "retained-render"), b"secret", current_time_millis()).await; + let manifest = |first: u64| { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n"); + for seq in first..first + 3 { + writeln!(body, "#EXTINF:4,\n{origin}/{seq}.ts").unwrap(); + } + normal_manifest(&body) + }; + let old_rendered_at = current_time_millis().saturating_sub(1000); + let (old_head, new_head) = { + let mut session = session.write().await; + let startup = session.startup.take(); + session.apply_origin_manifest(&manifest(100)).unwrap(); + for entry in session.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: old_rendered_at }; + } + let old_head = session.render_and_store_manifest(old_rendered_at).unwrap().first_proxy_seq; + session.apply_origin_manifest(&manifest(103)).unwrap(); + session.startup = startup; + let new_head = session.publishable_origin_head_proxy_seq.unwrap(); + while session.segment_prefetch_queue.pop_next().is_some() {} + let candidates = session + .segments + .iter() + .filter(|(_, entry)| !matches!(entry.status, SegmentCacheStatus::Ready { .. })) + .map(|(seq, _)| *seq) + .collect::>(); + for seq in candidates { + session.segments.get_mut(&seq).unwrap().status = SegmentCacheStatus::Discovered; + assert!(session.queue_segment_fetch_candidate( + seq, + tuliprox_hls::api::SegmentFetchPriority::Demand, + current_time_millis() + )); + } + (old_head, new_head) + }; + let session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("waiting-reused-session".into()); + manager + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("test-user", "render-gate"), + session_id, + "test-user".into(), + "render-session".into(), + 1, + "retained-render".into(), + 1, + current_time_millis(), + 60_000, + )) + .await; + let client = reqwest::Client::new(); + manager + .segment_worker_pool() + .wake_scheduler( + SegmentFetchContext { + session: session.clone(), + segment_cache: manager.segment_cache().clone(), + segment_repair: manager.segment_repair().clone(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: client.clone(), + no_redirect_client: client, + use_manual_redirects: false, + origin_io: None, + }, + current_time_millis(), + ) + .await; + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if session + .read() + .await + .startup + .as_ref() + .unwrap() + .revisions + .values() + .any(|revision| revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire) > 0) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("head prefix"); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, old_head); + let request = { + let app = app.clone(); + let session = session.clone(); + let lease_id = lease_id.clone(); + tokio::spawn(async move { + try_test_hls_cached_manifest_response( + &app, + &session, + &lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: shared::model::HlsStripMode::Segments, value: 0 }, + None, + super::HlsCachedManifestOptions::initial(Duration::from_secs(2)) + .requiring_newer_manifest(old_rendered_at), + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(5)).await; + assert!(!request.is_finished()); + headers.send_replace(true); + let response = tokio::time::timeout(Duration::from_secs(1), request).await.unwrap().unwrap().expect("playlist"); + let answered_at = current_time_millis(); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + assert!(body.contains(&format!("{new_head:06}.ts"))); + let state = session.read().await; + let rendered = state.last_rendered_manifest.as_ref().unwrap(); + assert!(answered_at.saturating_sub(rendered.rendered_at_ms) <= 50, "release exceeded two poll intervals"); + assert!(state.startup.as_ref().unwrap().revisions.values().all(|revision| !revision.revision().is_complete())); + assert!(!*bodies.borrow()); + drop(state); + bodies.send_replace(true); + origin_task.abort(); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs index 4dffc61ac..d50e43d26 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs @@ -239,6 +239,7 @@ async fn hls_provider_exhausted_without_provisioning_returns_custom_manifest() { let response = super::super::hls_shared_provisioning_or_provider_exhausted_response( &app_state, + None, &session, "hls-user", &input, @@ -282,6 +283,7 @@ async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segmen let response = super::super::hls_shared_provisioning_timeline_manifest_response( &app_state, + None, &session, &access_lease_id, HlsAccessLeaseState::Activated, @@ -315,6 +317,7 @@ async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segmen let response = super::super::hls_shared_provisioning_timeline_manifest_response( &app_state, + None, &session, &access_lease_id, HlsAccessLeaseState::Activated, @@ -342,6 +345,7 @@ async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { super::super::hls_shared_provisioning_timeline_manifest_response( &app_state, + None, &session, &initial_lease_id, HlsAccessLeaseState::Activated, @@ -404,6 +408,7 @@ async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segment let strip = app_state.hls.proxy.strip(); super::super::hls_shared_provisioning_timeline_manifest_response( &app_state, + None, &session, &access_lease_id, HlsAccessLeaseState::Activated, diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs index 4d48a48fb..23eb3c37f 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs @@ -548,6 +548,7 @@ async fn hls_provider_exhausted_grace_hold_waits_for_grace_period_before_retry() let strip = app_state.hls.proxy.strip(); let resolution = super::super::hls_provider_connections_exhausted_manifest_resolution( &app_state, + None, &session, "hls-user", &input, @@ -599,6 +600,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { let grace = super::super::hls_provider_connections_exhausted_manifest_resolution( &fixture.app_state, + None, &session, "hls-user", &input, @@ -614,6 +616,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { let response = super::super::hls_provider_connections_exhausted_manifest_resolution( &fixture.app_state, + None, &session, "hls-user", &input, @@ -635,6 +638,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { assert_eq!(plan.segment_duration_ms, 10_027); let replay = super::super::hls_provider_connections_exhausted_manifest_resolution( &fixture.app_state, + None, &session, "hls-user", &input, diff --git a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs index c8cc4608e..27d14456e 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs @@ -99,7 +99,9 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) { let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await; let key = super::super::build_hls_origin_source(&input, "12345").session_key(); let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap(); - ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await; + let baseline_sequences: &[u64] = + if mode == HlsStartupMode::Conservative { &[0, 1, 2, 3, 4, 5] } else { &[3, 4, 5] }; + ready(&app, &session, baseline_sequences).await; let response = try_test_hls_cached_manifest_response( &app, &session, @@ -113,7 +115,7 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) { .expect("complete baseline is published to the playback lease"); assert_eq!(response.status(), StatusCode::OK); let first = String::from_utf8(response_body(response).await.to_vec()).unwrap(); - deliver(&app, &session, &first, &[1000]).await; + deliver(&app, &session, &first, &[1000 + baseline_sequences[0]]).await; tokio::time::timeout(WAIT, async { let proxy_id = session.read().await.proxy_session_id.clone(); loop { @@ -123,7 +125,7 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) { .access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis()) .await .unwrap(); - if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) { + if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(baseline_sequences[0]) { break; } tokio::task::yield_now().await; diff --git a/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs index 7cabb3ffc..17459c36e 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/same_account_resume.rs @@ -104,7 +104,12 @@ async fn deliver_origin_segment(app: &Arc, session: &HlsSessionHandle, let origin_url = { let s = session.read().await; let segment = &s.segments[&seq]; - assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window"); + assert!( + (head..head + 6).contains(&segment.origin_key.host_local_sequence), + "current origin window: expected {head}..{}; got {} at proxy {seq}; manifest={manifest}", + head + 6, + segment.origin_key.host_local_sequence + ); segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone() }; let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string(); diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/http.rs b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs index 20ef221e0..102b4d1e2 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/http.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs @@ -62,6 +62,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn try_test_hls_cached_manif } super::super::super::try_hls_cached_manifest_response( app_state, + None, session, access_lease_id, access_lease_state, diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs b/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs index 885da9022..2429a60a8 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs @@ -68,6 +68,7 @@ impl CanonicalOwnerHandoffFixture { ) -> super::super::super::HlsCanonicalOwnerHandoffContext<'_> { let (lease_id, issued_at_ms) = &self.leases[lease_index]; super::super::super::HlsCanonicalOwnerHandoffContext { + fingerprint: None, app_state: &self.app_state, proxy_session_id: &self.proxy_session_id, access_lease_id: lease_id, diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs index 3db6f87fd..23dba5e18 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs @@ -78,6 +78,7 @@ async fn completed_owner_resolves_new_lease_standalone_without_reusing_old_termi let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; let mut response = Box::pin(super::super::join_hls_canonical_manifest_owner( super::super::HlsCanonicalOwnerHandoffContext { + fingerprint: None, app_state: &app_state, proxy_session_id: &proxy_session_id, access_lease_id: &new_lease_id, diff --git a/backend/app/src/api/model/app_state/view.rs b/backend/app/src/api/model/app_state/view.rs index 765063e87..9b034d404 100644 --- a/backend/app/src/api/model/app_state/view.rs +++ b/backend/app/src/api/model/app_state/view.rs @@ -92,12 +92,6 @@ app_state_views! { app_config, recordings, events <- event_manager, http_clients, recording_capacity, } - /// The handles the HLS proxy needs: itself, plus provider allocation and - /// session accounting. - hls_ctx => crate::api::model::hls_cache::HlsCtx { - app_config, hls_proxy <- hls.proxy, active_provider, connection_manager, active_users, - } - /// The handles admission reads: it decides over connections and users. admission_ctx => tuliprox_session::admission::AdmissionCtx { app_config, active_users, connection_manager, @@ -116,3 +110,23 @@ app_state_views! { maintenance as std::sync::Arc, } } + +impl AppStateView for crate::api::model::hls_cache::HlsCtx { + fn from_app_state(state: &AppState) -> Self { + let ctx = Self { + app_config: state.app_config.clone(), + hls_proxy: state.hls.proxy.clone(), + active_provider: state.active_provider.clone(), + connection_manager: state.connection_manager.clone(), + active_users: state.active_users.clone(), + }; + ctx.attach_lease_user_accounting(); + ctx + } +} + +impl AppState { + /// The HLS runtime and its authoritative session-accounting owner. + #[must_use] + pub fn hls_ctx(&self) -> crate::api::model::hls_cache::HlsCtx { self.view() } +} diff --git a/backend/app/tests/hls_revision_recovery.rs b/backend/app/tests/hls_revision_recovery.rs new file mode 100644 index 000000000..872d8cb28 --- /dev/null +++ b/backend/app/tests/hls_revision_recovery.rs @@ -0,0 +1,330 @@ +//! Real Tuliprox process + a streaming origin. No application HTTP client is stubbed. +use std::{ + fmt::Write as _, + path::PathBuf, + process::Stdio, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + sync::watch, +}; + +const BODY: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +// A configured Postman member is shared between the process scenarios. +static SCENARIO: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +struct Fixture { + directory: tempfile::TempDir, + base: String, + process: tokio::process::Child, + origin: tokio::task::JoinHandle<()>, + release: watch::Sender, + release_retry: watch::Sender, + attempts: Arc, + _scenario: tokio::sync::MutexGuard<'static, ()>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.process.start_kill(); + self.origin.abort(); + } +} + +impl Fixture { + #[allow(clippy::too_many_lines, reason = "Keeps the isolated process and streaming origin fixture together.")] + async fn start() -> Self { + let scenario = SCENARIO.lock().await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin_base = format!("http://{}", listener.local_addr().unwrap()); + let manifest_base = + std::env::var("TULIPROX_HLS_RECOVERY_MANIFEST_BASE").unwrap_or_else(|_| origin_base.clone()); + if manifest_base != origin_base { + let response = reqwest::Client::new() + .post(format!("{manifest_base}/configure")) + .json(&serde_json::json!({"segmentOrigin": origin_base})) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 204); + } + let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"recovery\" group-title=\"Test\",Recovery Channel\n{manifest_base}/index.m3u8\n"); + let mut manifest = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n".to_string(); + for seq in 100..106 { + writeln!(manifest, "#EXTINF:4,\n{origin_base}/{seq}.ts").unwrap(); + } + let attempts = Arc::new(AtomicUsize::new(0)); + let (release, released) = watch::channel(false); + let (release_retry, retry_released) = watch::channel(false); + let counted = attempts.clone(); + let origin = tokio::spawn(async move { + let mut children = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let (socket, _) = accepted.unwrap(); + let catalog = catalog.clone(); let manifest = manifest.clone(); + let attempts = counted.clone(); let mut released = released.clone(); + let mut retry_released = retry_released.clone(); + children.spawn(async move { + let mut reader = BufReader::new(socket); + let mut request = String::new(); + reader.read_line(&mut request).await.unwrap(); + let path = request.split_whitespace().nth(1).unwrap().to_owned(); + loop { + let mut line = String::new(); reader.read_line(&mut line).await.unwrap(); + if line == "\r\n" || line.is_empty() { break; } + } + let mut socket = reader.into_inner(); + if path == "/catalog.m3u" || path == "/index.m3u8" { + let body = if path == "/index.m3u8" { manifest } else { catalog }; + let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/vnd.apple.mpegurl\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(headers.as_bytes()).await.unwrap(); + socket.write_all(body.as_bytes()).await.unwrap(); + return; + } + // The accelerated entry is 103,104,105. Inject the + // failure into its first visible successor. + let failed_attempt = path == "/104.ts" && attempts.fetch_add(1, Ordering::AcqRel) == 0; + let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: video/mp2t\r\nConnection: close\r\n\r\n", BODY.len()); + socket.write_all(headers.as_bytes()).await.unwrap(); + if failed_attempt { + socket.write_all(b"stale-prefix").await.unwrap(); + // Hold an incomplete 200 open beyond the configured body idle timeout. + tokio::time::sleep(Duration::from_secs(2)).await; + return; + } + if path == "/104.ts" { + while !*retry_released.borrow_and_update() { + if retry_released.changed().await.is_err() { return; } + } + } + socket.write_all(&BODY[..188]).await.unwrap(); + while !*released.borrow_and_update() { + if released.changed().await.is_err() { return; } + } + let _ = socket.write_all(&BODY[188..]).await; + }); + } + _ = children.join_next(), if !children.is_empty() => {} + } + } + }); + let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = reserved.local_addr().unwrap().port(); + drop(reserved); + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + for name in ["web", "storage", "backup"] { + std::fs::create_dir(root.join(name)).unwrap(); + } + let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n origin_segment_timeout_ms: 300\n max_concurrent_segment_fetches_per_session: 3\n initial_manifest_wait_timeout_secs: 3\n startup:\n mode: optimistic\n segment_repair:\n max_level: off\n", root.join("web").display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display()); + let sources = format!("inputs:\n - name: recovery-origin\n type: m3u\n url: {origin_base}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [recovery-origin]\n targets:\n - name: recovery\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n"); + let users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Recovery test\nuser:\n - target: recovery\n credentials:\n - username: viewer\n password: test-password\n proxy: reverse\n server: local\n status: Active\n max_connections: 1\n"); + for (name, body) in [("config.yml", config), ("source.yml", sources), ("api-proxy.yml", users)] { + std::fs::write(root.join(name), body).unwrap(); + } + let log = std::fs::File::create(root.join("sut.log")).unwrap(); + let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox")) + .args(["-s", "-c", "config.yml", "-i", "source.yml", "-a", "api-proxy.yml"]) + .current_dir(root) + .stdout(Stdio::from(log.try_clone().unwrap())) + .stderr(Stdio::from(log)) + .kill_on_drop(true) + .spawn() + .unwrap(); + Self { + directory, + base: format!("http://127.0.0.1:{port}"), + process, + origin, + release, + release_retry, + attempts, + _scenario: scenario, + } + } + + fn log(&self) -> String { std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() } + + async fn playback(&mut self, client: &reqwest::Client) -> String { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + assert!(self.process.try_wait().unwrap().is_none(), "SUT exited: {}", self.log()); + if let Ok(response) = + client.get(format!("{}/m3u?username=viewer&password=test-password", self.base)).send().await + { + if let Ok(body) = response.text().await { + if let Some(url) = body.lines().find(|line| line.starts_with("http")) { + return url.to_owned(); + } + } + } + assert!(tokio::time::Instant::now() < deadline, "catalog unavailable: {}", self.log()); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } +} + +fn resolve_media(base: &str, body: &str, index: usize) -> String { + let uri = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).nth(index).expect("media URI"); + reqwest::Url::parse(base).unwrap().join(uri).unwrap().to_string() +} + +#[tokio::test] +async fn incomplete_origin_200_recovers_same_segment_uri_without_a_playlist_refresh() { + let mut fixture = Fixture::start().await; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap(); + let entry = fixture.playback(&client).await; + let master = client.get(&entry).send().await.unwrap(); + assert!(master.status().is_success(), "entry: {}", fixture.log()); + let master_body = master.text().await.unwrap(); + let variant = resolve_media(&entry, &master_body, 0); + let playlist = client.get(&variant).send().await.unwrap(); + assert_eq!(playlist.status(), 200, "startup: {}", fixture.log()); + let playlist = playlist.text().await.unwrap(); + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:3\n"), "entry follows the current live edge"); + let segment = resolve_media(&variant, &playlist, 1); + assert_eq!(fixture.attempts.load(Ordering::Acquire), 1); + // Exercise unexposed successor recovery after consuming this lease's one + // progressive request. A successor selected as the first GET may now pipe + // its prefix and therefore cannot silently switch bytes after a body drop. + let head = client.get(resolve_media(&variant, &playlist, 0)).send().await.unwrap(); + assert_eq!(head.status(), 200, "head must stream before EOF: {}", fixture.log()); + fixture.release.send_replace(true); + assert_eq!(head.bytes().await.unwrap().as_ref(), BODY); + let waiting = client.get(&segment).send().await.unwrap(); + assert_eq!(waiting.status(), 503, "retry must be temporary: {}", fixture.log()); + assert!(waiting.headers().contains_key("retry-after")); + // Make the pending response independent of the origin retry's timing. + fixture.release_retry.send_replace(true); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + let response = client.get(&segment).send().await.unwrap(); + if response.status() == 200 { + assert_eq!(response.bytes().await.unwrap().as_ref(), BODY); + break; + } + assert_eq!(response.status(), 503, "same URI must not become 404 during recovery: {}", fixture.log()); + assert!(tokio::time::Instant::now() < deadline, "retry never recovered: {}", fixture.log()); + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert!(fixture.attempts.load(Ordering::Acquire) >= 2); + let refreshed = client.get(&variant).send().await.unwrap(); + assert_eq!(refreshed.status(), 200, "recovered binding must not conflict: {}", fixture.log()); + assert!(refreshed.text().await.unwrap().contains(reqwest::Url::parse(&segment).unwrap().path())); + for _ in 0..3 { + let replay = client.get(&segment).send().await.unwrap(); + assert_eq!(replay.status(), 200); + assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY); + } + assert_eq!(fixture.attempts.load(Ordering::Acquire), 2, "repaired GETs reuse the committed revision"); + assert!(fixture.log().contains("outcome=replacement_bound")); + // Retain run evidence only when requested; test configuration and credentials are synthetic. + if let Ok(destination) = std::env::var("TULIPROX_HLS_RECOVERY_LOG") { + std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap(); + } + fixture.process.kill().await.unwrap(); +} + +#[tokio::test] +async fn first_requested_successor_streams_before_origin_eof() { + let mut fixture = Fixture::start().await; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap(); + let entry = fixture.playback(&client).await; + let master = client.get(&entry).send().await.unwrap().text().await.unwrap(); + let variant = resolve_media(&entry, &master, 0); + let playlist = client.get(&variant).send().await.unwrap(); + assert_eq!(playlist.status(), 200, "startup: {}", fixture.log()); + let playlist = playlist.text().await.unwrap(); + // Skip both the playlist head and the deliberately failing first successor. + let segment = resolve_media(&variant, &playlist, 2); + let mut response = tokio::time::timeout(Duration::from_secs(2), client.get(&segment).send()) + .await + .expect("first successor GET must not wait for origin EOF") + .unwrap(); + assert_eq!(response.status(), 200, "successor: {}", fixture.log()); + assert!(!response.headers().contains_key("content-length")); + let prefix = response.chunk().await.unwrap().expect("progressive prefix"); + assert_eq!(prefix.as_ref(), &BODY[..188]); + assert!(!*fixture.release.borrow(), "origin EOF is still held"); + fixture.release.send_replace(true); + fixture.release_retry.send_replace(true); + let mut bytes = prefix.to_vec(); + while let Some(chunk) = response.chunk().await.unwrap() { + bytes.extend_from_slice(&chunk); + } + assert_eq!(bytes, BODY); + let replay = client.get(&segment).send().await.unwrap(); + assert_eq!(replay.status(), 200); + assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY); + if let Ok(destination) = std::env::var("TULIPROX_HLS_FIRST_REQUEST_LOG") { + std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap(); + } + fixture.process.kill().await.unwrap(); +} + +#[tokio::test] +async fn exposed_failure_allows_a_new_player_with_user_connection_limit_one() { + let mut fixture = Fixture::start().await; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap(); + let entry = fixture.playback(&client).await; + let master = client.get(&entry).send().await.unwrap().error_for_status().unwrap().text().await.unwrap(); + let variant = resolve_media(&entry, &master, 0); + let playlist = client.get(&variant).send().await.unwrap().error_for_status().unwrap().text().await.unwrap(); + let segment = resolve_media(&variant, &playlist, 1); + // The first full GET deliberately selects the successor, exercising the + // integrated piping policy and exposing this attempt's prefix. + let exposed = client.get(&segment).send().await.unwrap(); + assert_eq!(exposed.status(), 200, "first successor GET must pipe: {}", fixture.log()); + fixture.release.send_replace(true); + assert!(exposed.bytes().await.is_err(), "exposed truncated body must not become successful EOF"); + let ended = client.get(&segment).send().await.unwrap(); + assert_eq!(ended.status(), 404, "old lease must end: {}", fixture.log()); + fixture.release_retry.send_replace(true); + let new_player = + reqwest::Client::builder().user_agent("new-reentry-player").timeout(Duration::from_secs(10)).build().unwrap(); + let master = new_player.get(&entry).send().await.unwrap(); + assert_eq!(master.status(), 200, "counted user reservation blocked reentry: {}", fixture.log()); + let body = master.text().await.unwrap(); + assert!(body.contains("#EXT-X-STREAM-INF"), "new entry must reach live playback: {body}"); + let new_variant = resolve_media(&entry, &body, 0); + assert_ne!(new_variant, variant); + let previous_session = reqwest::Url::parse(&variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned(); + let current_session = + reqwest::Url::parse(&new_variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned(); + assert_eq!(previous_session, current_session, "the shared origin session stays alive"); + // Reentry releases the user slot before the origin retry finishes. The + // pending lease may still wait for that retry's publishable media. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let playlist = loop { + let response = new_player.get(&new_variant).send().await.unwrap(); + if response.status() == 200 { + break response; + } + // Before its first publication, a lease can use the unavailable-channel + // fallback (404 without a configured custom stream) or a temporary 503. + assert!( + matches!(response.status().as_u16(), 404 | 503), + "new live playlist rejected with {}: {}", + response.status(), + fixture.log() + ); + assert!(tokio::time::Instant::now() < deadline, "new live playlist never recovered: {}", fixture.log()); + tokio::time::sleep(Duration::from_millis(25)).await; + }; + let new_segment = resolve_media(&new_variant, &playlist.text().await.unwrap(), 1); + let repaired = new_player.get(new_segment).send().await.unwrap(); + assert_eq!(repaired.status(), 200, "new player cannot use the successful revision: {}", fixture.log()); + assert_eq!(repaired.bytes().await.unwrap().as_ref(), BODY); + assert_eq!(client.get(&segment).send().await.unwrap().status(), 404); + fixture.process.kill().await.unwrap(); +} diff --git a/backend/hls/src/gc/reclaim.rs b/backend/hls/src/gc/reclaim.rs index c3f38164e..dd396f3d1 100644 --- a/backend/hls/src/gc/reclaim.rs +++ b/backend/hls/src/gc/reclaim.rs @@ -4,8 +4,9 @@ use super::{ selection::{ capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment, fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate, - remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries, - session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids, + remove_map_entry, remove_obsolete_deferred_heads, remove_obsolete_segment_metadata, remove_segment_entry, + remove_stale_queue_entries, session_cache_size, take_expired_transient_objects, + unprotected_unreferenced_map_ids, }, CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle, @@ -203,6 +204,7 @@ impl HlsGarbageCollector { report.stale_queue_entries_removed = report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); + remove_obsolete_segment_metadata(session, &protected_set_for_cache_reclamation(session, cache, None, None)); remove_obsolete_deferred_heads(session, cache, None, None, deletions); let transient_before = session.transient.resources.len(); let protected = protected_set_for_cache_reclamation(session, cache, None, None); diff --git a/backend/hls/src/gc/revisions.rs b/backend/hls/src/gc/revisions.rs index c8e4d7aa3..a44b29957 100644 --- a/backend/hls/src/gc/revisions.rs +++ b/backend/hls/src/gc/revisions.rs @@ -75,6 +75,21 @@ impl HlsGarbageCollector { continue; }; let id = session.proxy_session_id.clone(); + if let Some(leases) = &mut lease_guard { + session.live_window_demands.retain(|lease_id, demand| { + leases.response_snapshot(lease_id, &id, tuliprox_core::utils::current_time_millis()).is_some_and( + |lease| { + lease.issued_at_ms == demand.issued_at_ms + && matches!( + lease.state, + crate::HlsAccessLeaseState::Pending + | crate::HlsAccessLeaseState::Activated + | crate::HlsAccessLeaseState::Idle + ) + }, + ) + }); + } let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq); let crate::HlsSession { startup, segments, .. } = &mut *session; if let Some(startup) = startup { diff --git a/backend/hls/src/gc/selection.rs b/backend/hls/src/gc/selection.rs index 43ee9f261..10d71d2bd 100644 --- a/backend/hls/src/gc/selection.rs +++ b/backend/hls/src/gc/selection.rs @@ -147,13 +147,36 @@ pub(super) fn remove_stale_queue_entries(session: &mut HlsSession) -> usize { removed } +/// Fast-start origin mappings are needed only while their origin window or an +/// existing publication, reader or cache mutation still owns them. +pub(super) fn remove_obsolete_segment_metadata(session: &mut HlsSession, protected: &ProtectedSet) { + if !session.uses_fast_start_live_window() { + return; + } + let Some(origin) = session.origin_window_proxy_seqs.as_ref() else { return }; + let origin = origin.iter().copied().collect::>(); + let obsolete = session + .segments + .iter() + .filter_map(|(seq, entry)| { + (matches!(entry.status, SegmentCacheStatus::Discovered) + && !origin.contains(seq) + && !protected.segment_proxy_seqs.contains(seq)) + .then_some(*seq) + }) + .collect::>(); + for seq in obsolete { + let _ = remove_segment_entry(session, seq); + } +} + pub(super) fn duration_expired_head_segment( session: &HlsSession, protected: &ProtectedSet, policy: &GarbageCollectionPolicy, now_ms: u64, ) -> Option { - let (proxy_seq, segment) = session.segments.iter().next()?; + let (proxy_seq, segment) = fifo_head_segment(session)?; if protected.segment_proxy_seqs.contains(proxy_seq) { return None; } @@ -177,7 +200,7 @@ pub(super) fn fifo_head_size_candidate( session: &HlsSession, protected: &ProtectedSet, ) -> Option { - let (proxy_seq, segment) = session.segments.iter().next()?; + let (proxy_seq, segment) = fifo_head_segment(session)?; if protected.segment_proxy_seqs.contains(proxy_seq) { return None; } @@ -191,6 +214,12 @@ pub(super) fn fifo_head_size_candidate( }) } +/// Origin mappings without cache objects are not FIFO barriers. All other +/// statuses retain the existing protection and expiry checks at the media head. +fn fifo_head_segment(session: &HlsSession) -> Option<(&u64, &super::super::SegmentEntry)> { + session.segments.iter().find(|(_, segment)| !matches!(segment.status, SegmentCacheStatus::Discovered)) +} + fn segment_last_relevant_at_ms(segment: &super::super::SegmentEntry) -> Option { let status_at = match segment.status { SegmentCacheStatus::Ready { ready_at_ms, .. } => Some(ready_at_ms), @@ -276,19 +305,30 @@ pub(super) fn capacity_reclamation_evidence( pub(super) fn remove_segment_entry(session: &mut HlsSession, proxy_seq: u64) -> Option { let segment = session.segments.remove(&proxy_seq)?; + // Media outside the selected window can be evicted, but the current + // origin's mapping must survive until its next slide. Otherwise an older, + // unselected URI is appended anew ahead of already published overlaps. + let retain_origin_metadata = + session.uses_fast_start_live_window() && session.live_origin_sequences().contains(&proxy_seq); if let Some(startup) = &mut session.startup { startup.revisions.remove(&proxy_seq); } let removed_ready_media = matches!(segment.status, SegmentCacheStatus::Ready { .. }); session.segment_prefetch_queue.remove(proxy_seq); - session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq); - if segment.discontinuity_before { + if retain_origin_metadata { + let mut metadata = segment.clone(); + metadata.status = SegmentCacheStatus::Discovered; + session.segments.insert(proxy_seq, metadata); + } else { + session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq); + } + if segment.discontinuity_before && !retain_origin_metadata { session.discontinuity_sequence = session.discontinuity_sequence.saturating_add(1); } if removed_ready_media { session.advance_media_readiness_generation(); } - if session.publishable_origin_head_proxy_seq == Some(proxy_seq) { + if session.publishable_origin_head_proxy_seq == Some(proxy_seq) && !retain_origin_metadata { session.publishable_origin_head_proxy_seq = session .segments .range(proxy_seq.saturating_add(1)..) diff --git a/backend/hls/src/gc/tests/publication.rs b/backend/hls/src/gc/tests/publication.rs index 051327940..341a74343 100644 --- a/backend/hls/src/gc/tests/publication.rs +++ b/backend/hls/src/gc/tests/publication.rs @@ -292,3 +292,317 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { assert!(remove_segment_entry(&mut session, 7).is_some()); assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500); } + +fn install_fast_start(session: &mut HlsSession) { + let config = tuliprox_core::model::HlsStartupConfig { + mode: shared::model::HlsStartupMode::Optimistic, + ..Default::default() + }; + session.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 1_000, + worker: std::sync::Weak::new(), + access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + revisions: std::collections::BTreeMap::new(), + policy_fixed: true, + store: Arc::new(crate::SegmentRevisionStore::default()), + budget: crate::ProgressiveBudgetManager::new(config), + }); +} + +#[test] +fn metadata_only_prefix_does_not_block_reclaimable_ready_segments() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "review-fifo-prefix"), b"secret", 0); + session.apply_origin_manifest(&six_segment_manifest()).unwrap(); + install_fast_start(&mut session); + for seq in [3, 4, 5] { + session.segments.get_mut(&seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: 0 }; + } + session.apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:7\n#EXTINF:4,\n7.ts\n#EXTINF:4,\n8.ts\n#EXTINF:4,\n9.ts\n#EXTINF:4,\n10.ts\n#EXTINF:4,\n11.ts\n#EXTINF:4,\n12.ts\n" + )).unwrap(); + let protected = ProtectedSet::from_session(&session); + assert!(!protected.segment_proxy_seqs.contains(&3)); + assert!(matches!(session.segments[&0].status, SegmentCacheStatus::Discovered)); + let reclaim = crate::gc::selection::fifo_head_size_candidate(&session, &protected); + assert_eq!( + reclaim.map(|entry| entry.proxy_seq), + Some(3), + "unfetched metadata 0..2 must not prevent reclamation of expired media 3..5" + ); + assert_eq!( + crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000), + Some(3) + ); + // The first actual media object still blocks FIFO if it is protected or unexpired. + assert_eq!(crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 0), None); + let mut protected = protected; + protected.segment_proxy_seqs.insert(3); + assert!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).is_none()); + assert_eq!( + crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000), + None + ); +} + +#[test] +fn reinserted_metadata_does_not_block_subsequent_cache_deletions() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "review-fifo-delete"), b"secret", 0); + session.apply_origin_manifest(&six_segment_manifest()).unwrap(); + install_fast_start(&mut session); + for entry in session.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: 0 }; + } + let protected = ProtectedSet::from_session(&session); + assert_eq!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).unwrap().proxy_seq, 0); + remove_segment_entry(&mut session, 0).unwrap(); + let protected = ProtectedSet::from_session(&session); + assert!(!protected.segment_proxy_seqs.contains(&1)); + let next = crate::gc::selection::fifo_head_size_candidate(&session, &protected); + assert_eq!( + next.map(|entry| entry.proxy_seq), + Some(1), + "preserving origin mapping must not prevent further capacity reclamation" + ); + assert_eq!( + crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000), + Some(1) + ); + for seq in [1, 2] { + remove_segment_entry(&mut session, seq).unwrap(); + assert!(matches!(session.segments[&seq].status, SegmentCacheStatus::Discovered)); + } + let protected = ProtectedSet::from_session(&session); + assert!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).is_none()); + assert_eq!( + crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000), + None + ); +} + +#[tokio::test] +async fn fast_start_gc_reclaims_multiple_objects_while_retaining_origin_mappings() { + for size_pressure in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let (gc, session) = gc_with_session(&directory).await; + update_gc_policy(&gc, |policy| { + policy.session_idle_timeout_ms = u64::MAX; + if size_pressure { + policy.cache_bytes_per_session = 36; + } + }); + let keys = { + let mut state = session.write().await; + state.apply_origin_manifest(&six_segment_manifest()).unwrap(); + install_fast_start(&mut state); + for entry in state.segments.values_mut() { + gc.cache.write_bytes_and_commit(&entry.cache_key, b"segment-body").await.unwrap(); + entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + state.segments.values().map(|entry| entry.cache_key.clone()).collect::>() + }; + let report = gc.run_once(if size_pressure { 0 } else { 100_000 }).await.unwrap(); + assert_eq!(report.segments_deleted_duration, if size_pressure { 0 } else { 3 }); + assert_eq!(report.segments_deleted_size_session, if size_pressure { 3 } else { 0 }); + let state = session.read().await; + assert_eq!(state.origin_to_proxy.len(), 6); + for seq in 0..3 { + assert!(matches!(state.segments[&seq].status, SegmentCacheStatus::Discovered)); + } + for (index, key) in keys.iter().enumerate() { + assert_eq!(gc.cache.metadata(key).await.unwrap().is_some(), index >= 3); + } + drop(state); + session.write().await.apply_origin_manifest(&rolling_metadata_manifest(7)).unwrap(); + gc.run_once(0).await.unwrap(); + let state = session.read().await; + for seq in 0..3 { + assert!(!state.segments.contains_key(&seq)); + assert!(!state.origin_to_proxy.values().any(|mapped| *mapped == seq)); + } + assert!((6..12).all(|seq| state.segments.contains_key(&seq))); + } +} + +fn rolling_metadata_manifest(first: u64) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n"); + for seq in first..first + 6 { + writeln!(body, "#EXTINF:4,\n{seq}.ts").unwrap(); + } + normal_manifest(&body) +} + +#[tokio::test] +async fn obsolete_metadata_stays_bounded_across_one_hundred_live_entries() { + use shared::model::{HlsCacheConfigDto, HlsStartupMode, Secs}; + use tuliprox_core::model::HlsCacheConfig; + + for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let mut dto = HlsCacheConfigDto { + cache_path: Some(directory.path().to_string_lossy().into_owned()), + cache_duration: Secs::new(1), + session_idle_timeout: Secs::new(1_000_000), + ..Default::default() + }; + dto.startup.mode = mode; + let manager = crate::HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "metadata-soak"), b"secret", 0).await; + for round in 0..100 { + let first = round * 6; + let keys = { + let mut state = session.write().await; + state.apply_origin_manifest_for_host(&rolling_metadata_manifest(first), 1).unwrap(); + let plan = state.plan_live_window(crate::live_window::HlsLiveWindowView::Entry).unwrap(); + assert_eq!(plan.visible, vec![first + 3, first + 4, first + 5]); + plan.visible + .iter() + .map(|seq| { + let entry = state.segments.get_mut(seq).unwrap(); + entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + entry.cache_key.clone() + }) + .collect::>() + }; + for key in keys { + manager.segment_cache().write_bytes_and_commit(&key, b"segment-body").await.unwrap(); + } + manager.run_garbage_collection_once(100_000 + round).await.unwrap(); + let mut state = session.write().await; + assert_eq!(state.segments.keys().copied().collect::>(), (first..first + 6).collect::>()); + assert_eq!(state.origin_to_proxy.len(), 6); + assert!(state.origin_to_proxy.values().all(|seq| (first..first + 6).contains(seq))); + assert_eq!( + state.segments.values().filter(|entry| matches!(entry.status, SegmentCacheStatus::Discovered)).count(), + 3 + ); + // Refreshing the same origin must reuse even its unfetched mappings. + state.apply_origin_manifest_for_host(&rolling_metadata_manifest(first), 1).unwrap(); + assert_eq!(state.live_origin_sequences(), (first..first + 6).collect::>()); + assert_eq!(state.proxy_next_seq, Some(first + 6)); + } + } +} + +#[tokio::test] +async fn obsolete_metadata_is_removed_only_after_publication_and_reader_protection_release() { + let directory = tempfile::tempdir().unwrap(); + let (gc, session) = gc_with_session(&directory).await; + let lease_id = HlsAccessLeaseId("metadata-owner".into()); + { + let mut state = session.write().await; + state.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap(); + for entry in state.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + state.render_and_store_manifest(0).unwrap(); + install_fast_start(&mut state); + for entry in state.segments.values_mut() { + entry.status = SegmentCacheStatus::Discovered; + } + state.apply_origin_manifest(&rolling_metadata_manifest(6)).unwrap(); + state.segments.get_mut(&0).unwrap().access.reader_started(0); + state.live_window_demands.insert( + lease_id.clone(), + crate::live_window::HlsLiveWindowDemand { + issued_at_ms: 0, + view: crate::live_window::HlsLiveWindowView::Entry, + published: vec![1], + }, + ); + state.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: HlsTerminalTailGeneration(1), + base_proxy_seqs: Arc::from([2]), + key_bindings: Arc::from([]), + }, + ); + } + gc.run_once(0).await.unwrap(); + assert_eq!(session.read().await.segments.len(), 12); + session.write().await.last_rendered_manifest = None; + gc.run_once(0).await.unwrap(); + { + let mut state = session.write().await; + assert_eq!(state.segments.keys().copied().collect::>(), vec![0, 1, 2, 6, 7, 8, 9, 10, 11]); + assert_eq!(state.origin_to_proxy.len(), 9); + state.segments.get_mut(&0).unwrap().access.reader_finished(); + assert!(state.live_window_demands.remove(&lease_id).is_some()); + assert!(state.remove_terminal_tail_protection(&lease_id).is_some()); + } + gc.run_once(0).await.unwrap(); + let state = session.read().await; + assert_eq!(state.segments.keys().copied().collect::>(), (6..12).collect::>()); + assert_eq!(state.origin_to_proxy.len(), 6); +} + +#[test] +fn obsolete_metadata_cleanup_preserves_non_fast_start_timeline_mappings() { + for archive in [false, true] { + let mut session = HlsSession::new(HlsSessionKey::new(1, "retained-metadata"), b"secret", 0); + session.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap(); + if archive { + install_fast_start(&mut session); + session.origin_source.archive_reference = Some(1_784_898_000); + } + session.segments.get_mut(&0).unwrap().discontinuity_before = true; + session.discontinuity_sequence = 7; + session.apply_origin_manifest(&rolling_metadata_manifest(6)).unwrap(); + assert!(!session.uses_fast_start_live_window()); + let protected = ProtectedSet::from_session(&session); + assert!(!protected.segment_proxy_seqs.contains(&0)); + let mappings = session.origin_to_proxy.clone(); + let sequences = session.segments.keys().copied().collect::>(); + let head = session.publishable_origin_head_proxy_seq; + let next = session.proxy_next_seq; + + crate::gc::selection::remove_obsolete_segment_metadata(&mut session, &protected); + + assert_eq!(session.origin_to_proxy, mappings, "archive={archive}"); + assert_eq!(session.segments.keys().copied().collect::>(), sequences); + assert_eq!(session.discontinuity_sequence, 7); + assert_eq!(session.publishable_origin_head_proxy_seq, head); + let mut overlapping = rolling_metadata_manifest(0); + overlapping.segments.extend(rolling_metadata_manifest(6).segments); + session.apply_origin_manifest(&overlapping).unwrap(); + assert_eq!(session.proxy_next_seq, next, "an overlapping URI must retain its original proxy sequence"); + } +} + +#[test] +fn obsolete_metadata_cleanup_requires_origin_evidence_and_preserves_other_cache_states() { + use crate::gc::{policy::protected_set_for_cache_reclamation, selection::remove_obsolete_segment_metadata}; + + let directory = tempfile::tempdir().unwrap(); + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let mut session = HlsSession::new(HlsSessionKey::new(1, "metadata-states"), b"secret", 0); + session.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap(); + install_fast_start(&mut session); + session.origin_window_proxy_seqs = None; + remove_obsolete_segment_metadata(&mut session, &ProtectedSet::default()); + assert_eq!(session.segments.len(), 6); + session.origin_window_proxy_seqs = Some(Vec::new()); + for (seq, status) in [ + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }, + SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::Demand, started_at_ms: 0 }, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Demand, queued_at_ms: 0 }, + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 0 }, + SegmentCacheStatus::FailedRetryable { failed_at_ms: 0, retry_after_ms: 0 }, + ] + .into_iter() + .enumerate() + { + session.segments.get_mut(&u64::try_from(seq).unwrap()).unwrap().status = status; + } + let excluded = cache.object_path(&session.segments[&5].cache_key); + let protected = protected_set_for_cache_reclamation(&session, &cache, Some(&excluded), None); + remove_obsolete_segment_metadata(&mut session, &protected); + assert_eq!(session.segments.len(), 6); + let protected = protected_set_for_cache_reclamation(&session, &cache, None, None); + remove_obsolete_segment_metadata(&mut session, &protected); + assert!(!session.segments.contains_key(&5)); + assert_eq!(session.segments.len(), 5); + assert_eq!(session.origin_to_proxy.len(), 5); +} diff --git a/backend/hls/src/hls_ctx.rs b/backend/hls/src/hls_ctx.rs index 1da94670c..f7035a438 100644 --- a/backend/hls/src/hls_ctx.rs +++ b/backend/hls/src/hls_ctx.rs @@ -26,6 +26,8 @@ pub struct HlsCtx { } impl HlsCtx { + /// Give request-independent lease completion its authoritative user registry. + pub fn attach_lease_user_accounting(&self) { self.hls_proxy.bind_lease_user_accounting(&self.active_users); } /// The subset admission reads: it decides over connections and users, not /// over the cache. pub fn admission_ctx(&self) -> tuliprox_session::admission::AdmissionCtx { @@ -38,6 +40,7 @@ impl HlsCtx { /// A weak handle, for state the proxy itself owns. pub fn downgrade(&self) -> WeakHlsCtx { + self.attach_lease_user_accounting(); WeakHlsCtx { app_config: Arc::clone(&self.app_config), hls_proxy: Arc::downgrade(&self.hls_proxy), @@ -128,12 +131,14 @@ impl HlsCtx { None, )); - Self { + let ctx = Self { hls_proxy: Arc::new(crate::manager::HlsProxyManager::new()), app_config: Arc::clone(&app_config), active_provider, connection_manager, active_users, - } + }; + ctx.attach_lease_user_accounting(); + ctx } } diff --git a/backend/hls/src/lease.rs b/backend/hls/src/lease.rs index 46c859e18..8d7308637 100644 --- a/backend/hls/src/lease.rs +++ b/backend/hls/src/lease.rs @@ -327,11 +327,13 @@ mod identity; mod lifecycle; mod publication; mod revision_bindings; +mod revision_recovery; mod runtime_policy; mod terminal; mod timing; pub use self::identity::new_hls_access_lease_id; +pub(crate) use self::revision_recovery::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome}; use self::{ error::HlsAvailabilityEvidenceAdvanceError, identity::HlsMediaLeasePlaybackIdentity, diff --git a/backend/hls/src/lease/identity.rs b/backend/hls/src/lease/identity.rs index 5908eecec..5e5b24d25 100644 --- a/backend/hls/src/lease/identity.rs +++ b/backend/hls/src/lease/identity.rs @@ -26,6 +26,12 @@ pub(super) enum HlsMediaLeasePlaybackIdentity { impl HlsLeaseManifestPublicationGuard { pub(crate) const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms } + pub(crate) const fn media_identity(self) -> HlsMediaLeaseIdentity { + HlsMediaLeaseIdentity { + issued_at_ms: self.issued_at_ms, + playback: HlsMediaLeasePlaybackIdentity::Live { admission_generation: self.admission_generation }, + } + } } impl HlsAccessLease { diff --git a/backend/hls/src/lease/publication.rs b/backend/hls/src/lease/publication.rs index c3dd770a8..a1b8419c5 100644 --- a/backend/hls/src/lease/publication.rs +++ b/backend/hls/src/lease/publication.rs @@ -202,4 +202,6 @@ impl HlsAccessLeaseStore { fn manifest_source_is_regressive(current: &HlsLeaseManifestSnapshot, candidate: &HlsLeaseManifestSnapshot) -> bool { candidate.source_commit_identity.commit_generation() < current.source_commit_identity.commit_generation() + || (candidate.startup_revisions.as_ref().is_some_and(|publication| publication.window_plan.is_some()) + && candidate.first_proxy_seq < current.first_proxy_seq) } diff --git a/backend/hls/src/lease/revision_bindings.rs b/backend/hls/src/lease/revision_bindings.rs index a76c7e3ec..893c43004 100644 --- a/backend/hls/src/lease/revision_bindings.rs +++ b/backend/hls/src/lease/revision_bindings.rs @@ -1,6 +1,6 @@ use super::{ - HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, - ProxySessionId, + lease_state_allows_use, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot, + HlsLeasePlaybackMode, ProxySessionId, }; use std::sync::Arc; @@ -50,14 +50,19 @@ impl HlsAccessLeaseStore { ) -> Option { self.by_lease_id.get(lease_id)?.revision_bindings.as_ref()?.revisions.get(&seq).cloned() } + /// Reserves the lease's single startup opportunity for its first authorized + /// full-segment request, including a cache hit. The caller excludes partial + /// ranges and holds this lock across lease-identity validation and lookup. + /// Bindings include still-retained URIs from earlier published windows. pub fn try_claim_progressive_startup(&mut self, lease_id: &HlsAccessLeaseId, seq: u64, now_ms: u64) -> bool { let Some(lease) = self.by_lease_id.get_mut(lease_id) else { return false; }; - if lease.playback_mode != HlsLeasePlaybackMode::Live + if !lease_state_allows_use(lease.state) + || lease.playback_mode != HlsLeasePlaybackMode::Live || lease.progressive_startup_claimed || lease.validity_due_at_ms() <= now_ms - || lease.last_manifest_snapshot.as_ref().is_none_or(|snapshot| snapshot.first_proxy_seq != seq) + || lease.revision_bindings.as_ref().is_none_or(|bindings| !bindings.revisions.contains_key(&seq)) { return false; } @@ -74,7 +79,9 @@ pub(super) fn commit_lease_revision_bindings(lease: &mut HlsAccessLease, snapsho || current.is_some_and(|current| { current.retired_through.is_some_and(|retired| *seq <= retired) || current.revisions.get(seq).is_some_and(|bound| { - bound != revision && !bound.revision().is_replaceable_in(publication.mode) + bound != revision + && (!bound.revision().is_replaceable_in(publication.mode) + || revision.revision().id() <= bound.revision().id()) }) }) }) { diff --git a/backend/hls/src/lease/revision_recovery.rs b/backend/hls/src/lease/revision_recovery.rs new file mode 100644 index 000000000..3f0e98cca --- /dev/null +++ b/backend/hls/src/lease/revision_recovery.rs @@ -0,0 +1,186 @@ +//! One transaction for repairing an already published segment binding. +use super::{ + HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsAccessLeaseState, HlsAccessLeaseStore, HlsLeasePlaybackMode, + HlsMediaLeaseIdentity, +}; +use crate::{HlsSession, SegmentCacheStatus, SegmentRevisionGuard, SegmentRevisionState}; +use std::sync::{atomic::Ordering, Arc}; + +#[derive(Debug, Clone)] +pub(crate) struct HlsRevisionRecoveryGuard { + identity: HlsMediaLeaseIdentity, + snapshot_generation: u64, + pub(crate) revision: SegmentRevisionGuard, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HlsRevisionRecoveryOutcome { + Unchanged, + ReplacementBound, + RetryPending, + ReentryRequired, +} + +impl HlsRevisionRecoveryGuard { + pub(crate) fn completion_identity(&self) -> (u64, u64) { + (self.identity.lease_issued_at_ms(), self.snapshot_generation) + } +} + +impl HlsAccessLeaseStore { + pub(crate) fn revision_recovery_guard( + &self, + lease_id: &HlsAccessLeaseId, + seq: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + Some(HlsRevisionRecoveryGuard { + identity: lease.media_identity()?, + snapshot_generation: lease.manifest_snapshot_generation, + revision: self.published_segment_revision(lease_id, seq)?, + }) + } + + fn revision_recovery_is_current( + &mut self, + lease_id: &HlsAccessLeaseId, + expected: &HlsRevisionRecoveryGuard, + session: &HlsSession, + now_ms: u64, + ) -> bool { + expected.identity.is_live() + && !session.is_gc_marked_for_removal() + && self.media_identity_is_current(lease_id, &session.proxy_session_id, expected.identity, now_ms) + && self.by_lease_id.get(lease_id).is_some_and(|lease| { + lease.manifest_snapshot_generation == expected.snapshot_generation + && self.published_segment_revision(lease_id, expected.revision.revision().key.proxy_seq).as_ref() + == Some(&expected.revision) + }) + } + + /// Caller holds the lease store and session locks in that order. The guard + /// fences the exact lease incarnation, snapshot and old revision together. + pub(crate) fn recover_revision_binding( + &mut self, + lease_id: &HlsAccessLeaseId, + expected: &HlsRevisionRecoveryGuard, + session: &HlsSession, + now_ms: u64, + ) -> HlsRevisionRecoveryOutcome { + use HlsRevisionRecoveryOutcome::{ReentryRequired, ReplacementBound, RetryPending, Unchanged}; + if !self.revision_recovery_is_current(lease_id, expected, session, now_ms) + || !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed) + { + return Unchanged; + } + let old = expected.revision.revision(); + let seq = old.key.proxy_seq; + if old.was_exposed() { + return ReentryRequired; + } + if !old.origin_binding_is_current(session) { + return ReentryRequired; + } + let mode = self.by_lease_id.get(lease_id).and_then(|lease| { + lease.last_manifest_snapshot.as_ref()?.startup_revisions.as_ref().map(|publication| publication.mode) + }); + if !mode.is_some_and(|mode| old.is_replaceable_in(mode)) { + return ReentryRequired; + } + let candidate = session.startup.as_ref().and_then(|startup| startup.revisions.get(&seq)); + let replacement = candidate.filter(|candidate| { + *candidate != &expected.revision + && candidate.revision().key.proxy_seq == seq + && candidate.revision().id() > old.id() + && candidate.revision().key.kind == old.key.kind + && candidate.revision().origin_binding_is_current(session) + && candidate.revision().is_complete() + && session + .segments + .get(&seq) + .is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. })) + }); + let Some(replacement) = replacement else { + return if session.segments.get(&seq).is_some_and(|entry| { + matches!( + entry.status, + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::CapacityDeferred { .. } + ) + }) { + RetryPending + } else { + ReentryRequired + }; + }; + if self.availability_evidence_can_advance().is_err() { + return RetryPending; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { return Unchanged }; + let Some(generation) = lease.manifest_snapshot_generation.checked_add(1) else { return RetryPending }; + let Some(bindings) = lease.revision_bindings.as_mut() else { return Unchanged }; + Arc::make_mut(bindings).revisions.insert(seq, replacement.clone()); + lease.manifest_snapshot_generation = generation; + if let Some(snapshot) = lease.last_manifest_snapshot.as_mut() { + snapshot.snapshot_generation = generation; + if let Some(publication) = snapshot.startup_revisions.as_mut() { + if Arc::make_mut(publication).revisions.contains_key(&seq) { + Arc::make_mut(publication).revisions.insert(seq, replacement.clone()); + } + } + } + let _ = self.advance_availability_evidence(&session.proxy_session_id); + log::debug!( + "HLS revision recovery: lease={} seq={seq} generation={generation} old={:?} new={:?} prefix={} exposed={} outcome=replacement_bound", + crate::safe_hls_access_lease_id(lease_id), old.id(), replacement.revision().id(), + old.prefix_available.load(Ordering::Acquire), old.was_exposed(), + ); + ReplacementBound + } + + /// An identity-checked, idempotent live -> ended transition. Removal and all + /// existing external cleanup happen through the manager's prepared removal. + pub(crate) fn end_revision_playback_if_current( + &mut self, + lease_id: &HlsAccessLeaseId, + expected: &HlsRevisionRecoveryGuard, + session: &HlsSession, + now_ms: u64, + ) -> Option { + if !self.revision_recovery_is_current(lease_id, expected, session, now_ms) + || !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed) + || self.availability_evidence_can_advance().is_err() + { + return None; + } + let lease = self.by_lease_id.get_mut(lease_id)?; + if lease.playback_mode != HlsLeasePlaybackMode::Live { + return None; + } + lease.end_playback(); + lease.state = HlsAccessLeaseState::Denied; + let _ = self.advance_availability_evidence(&session.proxy_session_id); + log::debug!( + "HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=reentry_required", + crate::safe_hls_access_lease_id(lease_id), expected.revision.revision().key.proxy_seq, + expected.snapshot_generation, expected.revision.revision().id(), + expected.revision.revision().prefix_available.load(Ordering::Acquire), expected.revision.revision().was_exposed(), + ); + self.prepare_access_lease_removal(lease_id) + } + + pub(crate) fn revision_bound_lease_ids(&self, session: &crate::ProxySessionId, seq: u64) -> Vec { + self.by_lease_id + .values() + .filter(|lease| { + lease.proxy_session_id == *session + && lease.playback_mode == HlsLeasePlaybackMode::Live + && lease.revision_bindings.as_ref().is_some_and(|bindings| bindings.revisions.contains_key(&seq)) + }) + .map(|lease| lease.lease_id.clone()) + .collect() + } +} diff --git a/backend/hls/src/lease/tests/behavior.rs b/backend/hls/src/lease/tests/behavior.rs index f783bc7ba..647216a44 100644 --- a/backend/hls/src/lease/tests/behavior.rs +++ b/backend/hls/src/lease/tests/behavior.rs @@ -532,6 +532,7 @@ fn published_uri_rejects_a_new_attempt_before_the_first_media_request() -> std:: } let mut first = manifest_snapshot(10); first.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 40, revisions: [(40, published.clone()), (41, tail.clone())].into_iter().collect(), @@ -545,6 +546,7 @@ fn published_uri_rejects_a_new_attempt_before_the_first_media_request() -> std:: next.revision().prefix_available.store(8, std::sync::atomic::Ordering::Release); let mut retry = manifest_snapshot(11); retry.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 40, revisions: [(40, next), (41, tail)].into_iter().collect(), diff --git a/backend/hls/src/lease/tests/startup.rs b/backend/hls/src/lease/tests/startup.rs index 89ff91062..6982f226d 100644 --- a/backend/hls/src/lease/tests/startup.rs +++ b/backend/hls/src/lease/tests/startup.rs @@ -12,6 +12,7 @@ fn retired_revision_bindings_cannot_be_rebound_after_the_hard_limit() -> std::io revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); let mut snapshot = manifest_snapshot(seq); snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 0, revisions: [(seq, revision)].into_iter().collect(), @@ -25,6 +26,7 @@ fn retired_revision_bindings_cannot_be_rebound_after_the_hard_limit() -> std::io new_attempt.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); let mut old_uri = manifest_snapshot(1031); old_uri.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 0, revisions: [(0, new_attempt)].into_iter().collect(), @@ -49,6 +51,7 @@ fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io:: } let mut snapshot = manifest_snapshot(generation + 1); snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 40, revisions: owners, @@ -66,20 +69,111 @@ fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io:: Ok(()) } -#[test] -fn startup_claim_only_accepts_the_published_head_once() { +fn startup_claim_fixture() -> std::io::Result<(HlsAccessLeaseStore, HlsAccessLeaseId)> { let session = ProxySessionId("proxy".into()); let lease_id = HlsAccessLeaseId("claim-lease".into()); let mut leases = HlsAccessLeaseStore::default(); leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000)); - assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(10), 2000).is_committed()); - assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100)); - if let Some(lease) = leases.by_lease_id.get_mut(&lease_id) { - lease.playback_cursor.first_requested_proxy_seq = Some(41); + let store = crate::SegmentRevisionStore::default(); + let mut revisions = std::collections::BTreeMap::new(); + for seq in [40, 41] { + let revision = store.create(session.clone(), seq, crate::SegmentRevisionKind::Raw)?; + revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + revisions.insert(seq, revision); } - assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2101)); + let mut snapshot = manifest_snapshot(10); + snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, + mode: shared::model::HlsStartupMode::Optimistic, + retained_start_seq: 40, + revisions, + })); + assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2000).is_committed()); + Ok((leases, lease_id)) +} + +#[test] +fn startup_claim_accepts_the_first_requested_published_segment_once() -> std::io::Result<()> { + let (mut leases, lease_id) = startup_claim_fixture()?; + assert!(!leases.try_claim_progressive_startup(&lease_id, 42, 2100), "unpublished URI cannot consume startup"); + assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101), "the player may start after the head"); assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2102)); assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2102)); + Ok(()) +} + +#[test] +fn startup_claim_accepts_a_retained_uri_after_the_playlist_head_moves() -> std::io::Result<()> { + let (mut leases, lease_id) = startup_claim_fixture()?; + let session = ProxySessionId("proxy".into()); + let mut snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot"); + snapshot.first_proxy_seq = 41; + snapshot.visible_segments = Arc::from([snapshot.visible_segments[1].clone()]); + Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).revisions.remove(&40); + assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed()); + assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2201), "an earlier published URI is still bound"); + Ok(()) +} + +#[test] +fn each_lease_gets_its_own_startup_claim_for_the_shared_revision() -> std::io::Result<()> { + let (mut leases, lease_id) = startup_claim_fixture()?; + let other_id = HlsAccessLeaseId("other-lease".into()); + let session = ProxySessionId("proxy".into()); + let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot"); + assert!(leases.prepare_access_lease(lease(other_id.clone(), &session.0, 2000))); + assert!(publish_manifest_snapshot(&mut leases, &other_id, &session, snapshot, 2100).is_committed()); + assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101)); + assert!(leases.try_claim_progressive_startup(&other_id, 41, 2102)); + assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2103)); + assert!(!leases.try_claim_progressive_startup(&other_id, 40, 2103)); + Ok(()) +} + +#[test] +fn startup_claim_cannot_be_revived_by_a_playlist_refresh_or_cursor_seek() -> std::io::Result<()> { + let (mut leases, lease_id) = startup_claim_fixture()?; + assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2100)); + let session = ProxySessionId("proxy".into()); + let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot"); + assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed()); + let _ = leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_cursor.record_request_started(100, 2201); + assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2202)); + Ok(()) +} + +#[test] +fn startup_claim_is_scoped_to_a_valid_live_lease() -> std::io::Result<()> { + for state in [super::HlsAccessLeaseState::Denied, super::HlsAccessLeaseState::Expired] { + let (mut leases, lease_id) = startup_claim_fixture()?; + leases.by_lease_id.get_mut(&lease_id).expect("lease").state = state; + assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100)); + } + let (mut leases, lease_id) = startup_claim_fixture()?; + assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 16_000)); + let (mut leases, lease_id) = startup_claim_fixture()?; + leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_mode = super::HlsLeasePlaybackMode::Ended; + assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100)); + Ok(()) +} + +#[tokio::test] +async fn concurrent_startup_claims_allow_exactly_one_request() -> std::io::Result<()> { + let (leases, lease_id) = startup_claim_fixture()?; + let leases = Arc::new(tokio::sync::RwLock::new(leases)); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let claim = |seq| { + let leases = Arc::clone(&leases); + let barrier = Arc::clone(&barrier); + let lease_id = lease_id.clone(); + tokio::spawn(async move { + barrier.wait().await; + leases.write().await.try_claim_progressive_startup(&lease_id, seq, 2100) + }) + }; + let (first, second) = tokio::join!(claim(40), claim(41)); + assert_ne!(first.expect("first task"), second.expect("second task")); + Ok(()) } #[test] @@ -89,6 +183,7 @@ fn optimistic_binding_accepts_pending_successors_but_not_failed_or_replaced_ones let snapshot_with = |revisions: std::collections::BTreeMap| { let mut snapshot = manifest_snapshot(1); snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Optimistic, retained_start_seq: 0, revisions, @@ -125,6 +220,7 @@ fn failed_unexposed_successor_may_be_rebound_but_exposed_bytes_never_change() -> let snapshot_with = |revisions: std::collections::BTreeMap| { let mut snapshot = manifest_snapshot(1); snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Optimistic, retained_start_seq: 0, revisions, diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index 2f09be4dd..ec5502bb7 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -51,6 +51,7 @@ mod ids; mod late_custom_response; mod lease; mod lifecycle; +mod live_window; mod manager; mod manifest_acceptance; mod manifest_commit; @@ -124,7 +125,7 @@ pub mod api { HlsTerminalResolution, }, availability_reevaluation::{HlsAvailabilityReevaluationObservation, HlsAvailabilityReevaluationRegistration}, - cache::{SegmentCacheKey, TransientObjectCacheKey}, + cache::{CachedSegmentMetadata, SegmentCacheKey, TransientObjectCacheKey}, deadline::hls_object_body_deadline, gc::exec_hls_cache_gc, headers::{ @@ -139,11 +140,12 @@ pub mod api { HlsFreshManifestRequiredReason, HlsLeaseStartupAdmissionState, HlsMediaLeaseIdentity, HlsPlaybackFamilyKey, }, lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, + live_window::{HlsLiveWindowPlan, HlsLiveWindowView}, manager::{exec_hls_lifecycle, HlsMediaActivityCommitOutcome, HlsProxyManager}, manifest_commit::{ hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request, - hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions, - HlsCommittedManifestBody, + hls_live_window_origin_allowed_for_request, hls_manifest_commit_requirement, + hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions, HlsCommittedManifestBody, }, manifest_fetch::{LiveHlsOriginEntry, RetryPolicy}, manifest_limits::{HlsManifestLimitViolation, MAX_HLS_MANIFEST_BYTES}, @@ -183,7 +185,7 @@ pub mod api { cold_start_retry_after_seconds, maybe_trigger_origin_refresh_with_outcome, trigger_origin_refresh_sync, HlsManifestCommitRequirement, HlsOriginRefreshTriggerOutcome, HlsPostRefreshRuntime, OriginRefreshRequest, }, - renderer::RenderedManifest, + renderer::{HlsManifestRenderer, RenderedManifest}, resource_fetch::{ retry_after_secs_from_ms, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsResourceFetchAttempt, }, diff --git a/backend/hls/src/live_window.rs b/backend/hls/src/live_window.rs new file mode 100644 index 000000000..3d0fdddbc --- /dev/null +++ b/backend/hls/src/live_window.rs @@ -0,0 +1,184 @@ +//! One metadata-only window plan for accelerated, untagged live playlists. +use crate::{ + media_reserve::HlsManifestCommitIdentity, HlsAccessLease, HlsAccessLeaseState, HlsSession, HlsSessionMode, +}; +use tuliprox_core::model::StripConfig; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsLiveWindowView { + Entry, + Active { first_published: u64 }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsLiveWindowPlan { + pub visible: Vec, + pub hidden_tail: Vec, + pub effective_strip_segments: usize, + pub origin_commits: u64, + pub origin_work_generation: u64, + pub source_commit_identity: Option, + origin_sequences: Vec, + strip: StripConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsLiveWindowPublication { + pub plan: HlsLiveWindowPlan, + pub body: std::sync::Arc, +} + +#[derive(Debug, Clone)] +pub(crate) struct HlsLiveWindowDemand { + pub(crate) issued_at_ms: u64, + pub(crate) view: HlsLiveWindowView, + pub(crate) published: Vec, +} + +impl HlsLiveWindowView { + pub fn for_lease(lease: &HlsAccessLease) -> Self { + if lease.state == HlsAccessLeaseState::Activated { + if let Some(snapshot) = &lease.last_manifest_snapshot { + return Self::Active { first_published: snapshot.first_proxy_seq }; + } + } + Self::Entry + } +} + +impl HlsLiveWindowPlan { + pub fn is_current(&self, session: &HlsSession) -> bool { + session.uses_fast_start_live_window() + && self.origin_commits == session.origin_manifest_commits() + && self.origin_work_generation == session.activity.origin_work_generation + && self.source_commit_identity == session.live_window_source_identity() + && self.origin_sequences.as_slice() == session.live_origin_sequences() + && self.strip == session.live_window_strip + } +} + +impl HlsSession { + pub(crate) fn live_origin_sequences(&self) -> &[u64] { + self.origin_window_proxy_seqs.as_deref().unwrap_or_default() + } + + pub(crate) fn live_window_source_identity(&self) -> Option { + if self + .live_window_origin_work_generation + .is_some_and(|generation| generation != self.activity.origin_work_generation) + { + return None; + } + self.live_window_origin_commit_identity.or_else(|| self.last_normal_manifest_commit_identity()) + } + + pub fn uses_fast_start_live_window(&self) -> bool { + self.startup.is_some() + && matches!(self.mode, HlsSessionMode::NormalCacheTimeline) + && self.origin_source.archive_reference.is_none() + && (self.origin_window_proxy_seqs.is_some() || self.segments.is_empty()) + && self.live_origin_sequences().iter().all(|seq| { + self.segments + .get(seq) + .is_none_or(|entry| entry.is_fast_start_eligible() && !crate::is_hls_provisioning_segment(entry)) + }) + } + + /// Entry selection uses only origin metadata. Active slides may retain cached + /// predecessors from this view to preserve three target durations. + pub fn plan_live_window(&self, view: HlsLiveWindowView) -> Option { + if !self.uses_fast_start_live_window() + || self.live_origin_sequences().is_empty() + || self + .live_window_origin_work_generation + .is_some_and(|generation| generation != self.activity.origin_work_generation) + { + return None; + } + let origin = self.live_origin_sequences(); + let (visible, hidden_tail, effective_strip_segments) = match view { + HlsLiveWindowView::Entry => { + let strip = crate::initial_strip::initial_hls_strip_segments_for_durations( + &self.live_window_strip, + &self.origin_window_durations_ms, + ); + let end = origin.len().saturating_sub(strip); + (origin[end.saturating_sub(3)..end].to_vec(), origin[end..].to_vec(), strip) + } + HlsLiveWindowView::Active { first_published } => { + let available = origin.iter().copied().filter(|seq| *seq >= first_published).collect::>(); + let mut visible = available[available.len().saturating_sub(6)..].to_vec(); + crate::renderer::retain_published_predecessors_from(self, &mut visible, first_published); + (visible, Vec::new(), 0) + } + }; + if visible.is_empty() { + return None; + } + Some(HlsLiveWindowPlan { + visible, + hidden_tail, + effective_strip_segments, + origin_commits: self.origin_manifest_commits(), + origin_work_generation: self.activity.origin_work_generation, + source_commit_identity: self.live_window_source_identity(), + origin_sequences: origin.to_vec(), + strip: self.live_window_strip.clone(), + }) + } + + pub fn plan_live_window_for_lease(&mut self, lease: &HlsAccessLease) -> Option { + let view = HlsLiveWindowView::for_lease(lease); + let plan = self.plan_live_window(view)?; + self.live_window_demands.insert( + lease.lease_id.clone(), + HlsLiveWindowDemand { + issued_at_ms: lease.issued_at_ms, + view, + published: lease + .last_manifest_snapshot + .as_ref() + .map_or_else(Vec::new, |snapshot| snapshot.visible_proxy_seqs().collect()), + }, + ); + Some(plan) + } + + pub(crate) fn required_live_windows(&self) -> Vec { + if self.live_window_demands.is_empty() { + return self.plan_live_window(HlsLiveWindowView::Entry).into_iter().collect(); + } + self.live_window_demands.values().filter_map(|demand| self.plan_live_window(demand.view)).collect() + } + + pub(crate) fn live_window_heads(&self) -> Vec { + self.required_live_windows().iter().filter_map(|plan| plan.visible.first().copied()).collect() + } + + pub(crate) fn record_live_window_publication( + &mut self, + lease_id: &crate::HlsAccessLeaseId, + snapshot: &crate::media_reserve::HlsLeaseManifestSnapshot, + ) { + if let Some(demand) = self.live_window_demands.get_mut(lease_id) { + demand.published = snapshot.visible_proxy_seqs().collect(); + } + self.longest_rendered_playlist_duration_ms = + self.longest_rendered_playlist_duration_ms.max(snapshot.playlist_duration_ms); + for seq in snapshot.visible_proxy_seqs() { + if let Some(entry) = self.segments.get_mut(&seq) { + entry.last_rendered_at_ms = Some(snapshot.delivered_at_ms); + if let Some(identity) = entry.media_resource_identity() { + self.published_resource_history.record(identity, seq); + } + if self.published_live_origin_baseline.is_none() { + self.published_live_origin_baseline = Some(crate::session::HlsPublishedLiveOriginBaseline { + evidence_proxy_seq: seq, + origin_epoch: entry.origin_key.origin_epoch, + rendered_at_ms: snapshot.delivered_at_ms, + }); + } + } + } + } +} diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index a97661bee..1ecde1a9b 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -77,6 +77,7 @@ pub struct HlsProxyManager { transient_resources: Arc, access_leases: Arc>, lifecycle: Arc, + lease_completion: Arc, account_overlap_cooldowns: Arc>>, metrics: Arc, qos: Arc, @@ -153,8 +154,10 @@ mod account_overlap; mod activity; mod cleanup; mod config; +mod lease_completion; mod lifecycle; mod publication; +mod revision_recovery; mod terminal; pub use self::lifecycle::exec_hls_lifecycle; @@ -162,6 +165,7 @@ pub use self::lifecycle::exec_hls_lifecycle; use self::lifecycle::hls_session_idle_protection_retry_at; #[cfg(test)] use self::publication::HlsRecoveryExecutionState; +pub(crate) use self::revision_recovery::HlsLeaseRuntime; use self::{ account_overlap::{HlsAccountOverlapCooldown, HlsAccountOverlapCooldownKey}, activity::hls_key_readiness_evidence_is_current, diff --git a/backend/hls/src/manager/config.rs b/backend/hls/src/manager/config.rs index c1e7e54a1..1b07935e0 100644 --- a/backend/hls/src/manager/config.rs +++ b/backend/hls/src/manager/config.rs @@ -111,7 +111,7 @@ impl HlsProxyManager { gc.install_revision_store(&revision_store); segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); - Self { + let manager = Self { sessions, segment_cache, segment_repair, @@ -135,6 +135,7 @@ impl HlsProxyManager { transient_resources: Arc::new(TransientResourceStore::new()), access_leases, lifecycle, + lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()), account_overlap_cooldowns, metrics, qos, @@ -146,7 +147,9 @@ impl HlsProxyManager { availability_reevaluations, terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), startup_observability: Arc::new(HlsStartupObservability::default()), - } + }; + manager.install_worker_revision_recovery(); + manager } pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() } @@ -198,10 +201,9 @@ impl HlsProxyManager { self.clear_runtime_cache_state_for_cache_path_change().await; } for session in self.sessions.list_sessions().await { - session - .write() - .await - .configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); + let mut session = session.write().await; + session.configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); + session.live_window_strip.clone_from(&runtime_config.strip); } self.segment_repair.update_config(hls_config.segment_repair.clone()); let global_fetch_semaphore = diff --git a/backend/hls/src/manager/lease_completion.rs b/backend/hls/src/manager/lease_completion.rs new file mode 100644 index 000000000..8f7101be2 --- /dev/null +++ b/backend/hls/src/manager/lease_completion.rs @@ -0,0 +1,202 @@ +//! Lease completion belongs to the runtime, not to the HTTP future waiting for it. +use super::{HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsProxyManager, ProxySessionId}; +use crate::{safe_proxy_session_id, sync_ext::MutexExt, HlsTerminalTailProtectionRemoval}; +use arc_swap::ArcSwapOption; +use log::debug; +use std::{ + collections::HashMap, + future::Future, + sync::{Arc, Mutex}, +}; +use tokio::sync::{watch, Notify}; +use tuliprox_session::ActiveUserManager; + +#[derive(Clone, PartialEq, Eq, Hash)] +pub(super) struct LeaseCompletionKey { + pub(super) lease_id: HlsAccessLeaseId, + pub(super) session_id: ProxySessionId, + pub(super) issued_at_ms: u64, + pub(super) snapshot_generation: Option, +} + +#[derive(Default)] +pub(super) struct HlsLeaseCompletion { + users: ArcSwapOption, + users_bound: Notify, + jobs: Mutex>>>, +} + +impl HlsLeaseCompletion { + fn bind_users(&self, users: &Arc) { + self.users.store(Some(Arc::clone(users))); + self.users_bound.notify_waiters(); + } + + pub(super) async fn users(&self) -> Arc { + loop { + let notified = self.users_bound.notified(); + if let Some(users) = self.users.load_full() { + return users; + } + // Before runtime attachment the lease remains its cleanup ticket; + // an absent accounting owner must never silently skip the release. + notified.await; + } + } + + pub(super) async fn run( + self: &Arc, + key: LeaseCompletionKey, + completion: impl Future + Send + 'static, + ) -> bool { + let mut receiver = { + let mut jobs = self.jobs.lock_unpoisoned(); + if let Some(receiver) = jobs.get(&key) { + receiver.clone() + } else { + let (sender, receiver) = watch::channel(None); + jobs.insert(key.clone(), receiver.clone()); + let owner = Arc::clone(self); + // Registered and spawned before the first await. Cancelling any + // waiter cannot cancel or erase the runtime's completion job. + tokio::spawn(async move { + let result = completion.await; + owner.jobs.lock_unpoisoned().remove(&key); + let _ = sender.send(Some(result)); + }); + receiver + } + }; + loop { + if let Some(result) = *receiver.borrow_and_update() { + return result; + } + if receiver.changed().await.is_err() { + return false; + } + } + } + + #[cfg(test)] + pub(super) fn pending_count(&self) -> usize { self.jobs.lock_unpoisoned().len() } +} + +impl HlsProxyManager { + /// Attach the server's authoritative user registry before serving leases. + pub fn bind_lease_user_accounting(&self, users: &Arc) { + self.lease_completion.bind_users(users); + } + + #[cfg(test)] + pub(crate) fn pending_lease_completions_for_test(&self) -> usize { self.lease_completion.pending_count() } +} + +impl super::HlsLeaseRuntime { + pub(super) async fn remove_prepared_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + preparation: &HlsAccessLeaseRemovalPreparation, + ) -> bool { + let key = LeaseCompletionKey { + lease_id: lease_id.clone(), + session_id: preparation.proxy_session_id.clone(), + issued_at_ms: preparation.issued_at_ms, + snapshot_generation: None, + }; + let runtime = self.clone(); + let lease_id = lease_id.clone(); + let preparation = preparation.clone(); + self.lease_completion + .run(key, async move { runtime.complete_prepared_lease(&lease_id, &preparation).await }) + .await + } +} + +impl super::HlsLeaseRuntime { + pub(super) async fn complete_prepared_lease( + &self, + lease_id: &HlsAccessLeaseId, + preparation: &crate::lease::HlsAccessLeaseRemovalPreparation, + ) -> bool { + // Hold the index and lease-store fences through external cleanup. A + // replacement with the same ID must not acquire QoS/repair ownership + // between removing the old incarnation and clearing those registries. + // Reject stale tickets before waiting for runtime attachment. + if !self.access_leases.read().await.prepare_access_lease_removal(lease_id).is_some_and(|current| { + current.proxy_session_id == preparation.proxy_session_id && current.issued_at_ms == preparation.issued_at_ms + }) { + return false; + } + let users = self.lease_completion.users().await; + let current = self.sessions.hold_current_proxy_session(&preparation.proxy_session_id).await; + let mut leases = self.access_leases.write().await; + let Some(lease) = leases.response_snapshot( + lease_id, + &preparation.proxy_session_id, + tuliprox_core::utils::current_time_millis(), + ) else { + return false; + }; + if lease.issued_at_ms != preparation.issued_at_ms { + return false; + } + users.release_session_streams_and_counted_reservation(&lease.username, &lease.user_session_token).await; + self.terminal_pending.cancel_lease(lease_id); + self.terminal_commit_retries.cancel_lease(lease_id); + if let Some(current) = current.as_ref() { + let session = current.session(); + let mut session = session.write().await; + if session + .live_window_demands + .get(lease_id) + .is_some_and(|demand| demand.issued_at_ms == preparation.issued_at_ms) + { + session.live_window_demands.remove(lease_id); + } + let finalized_manifest_released = + session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms); + if finalized_manifest_released { + debug!( + "HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id) + ); + } + if let Some(generation) = preparation.terminal_protection_generation { + let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation); + match removal { + HlsTerminalTailProtectionRemoval::Removed => { + debug!( + "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id), + generation.0 + ); + } + HlsTerminalTailProtectionRemoval::Missing => {} + HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { + debug!( + "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id), + generation.0, + actual.0 + ); + } + } + } else if session.remove_terminal_tail_protection(lease_id).is_some() { + debug!( + "HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id) + ); + } + } + self.standalone_custom_access.remove(lease_id); + self.segment_repair.remove_access_lease_window(lease_id).await; + self.startup_observability.remove_access_lease(lease_id); + self.qos.remove_access_lease(lease_id).await; + // Keep the exact lease present until all release steps have finished. + leases.remove_access_lease_if_preparation_matches(lease_id, preparation).is_some() + } +} diff --git a/backend/hls/src/manager/lifecycle.rs b/backend/hls/src/manager/lifecycle.rs index 733ea8dd3..1a9df6db3 100644 --- a/backend/hls/src/manager/lifecycle.rs +++ b/backend/hls/src/manager/lifecycle.rs @@ -51,8 +51,6 @@ impl HlsProxyManager { } pub(super) async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) { - self.terminal_pending.cancel_lease(lease_id); - self.terminal_commit_retries.cancel_lease(lease_id); let Some(preparation) = self.access_leases.read().await.prepare_access_lease_removal(lease_id) else { return; }; @@ -65,65 +63,15 @@ impl HlsProxyManager { } } - /// Removes one exact lease incarnation before releasing any media - /// protection owned by it. A stale preparation is intentionally - /// non-destructive because the replacement lease may reuse the same ID. + /// Complete one exact lease incarnation in the shared, request-independent + /// runtime. The lease is retired after accounting and media protection are + /// released; stale preparations cannot affect a replacement with the same ID. pub(super) async fn remove_prepared_access_lease( &self, lease_id: &HlsAccessLeaseId, preparation: &HlsAccessLeaseRemovalPreparation, ) -> bool { - let removed = - self.access_leases.write().await.remove_access_lease_if_preparation_matches(lease_id, preparation); - if removed.is_none() { - return false; - } - if let Some(session) = self.sessions.get_by_proxy_session_id(&preparation.proxy_session_id).await { - let mut session = session.write().await; - let finalized_manifest_released = - session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms); - if finalized_manifest_released { - debug!( - "HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed", - super::super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id) - ); - } - if let Some(generation) = preparation.terminal_protection_generation { - let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation); - match removal { - HlsTerminalTailProtectionRemoval::Removed => { - debug!( - "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed", - super::super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id), - generation.0 - ); - } - HlsTerminalTailProtectionRemoval::Missing => {} - HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { - debug!( - "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed", - super::super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id), - generation.0, - actual.0 - ); - } - } - } else if session.remove_terminal_tail_protection(lease_id).is_some() { - debug!( - "HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed", - super::super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id) - ); - } - } - self.standalone_custom_access.remove(lease_id); - self.segment_repair.remove_access_lease_window(lease_id).await; - self.startup_observability.remove_access_lease(lease_id); - self.qos.remove_access_lease(lease_id).await; - true + self.revision_recovery_runtime().remove_prepared_access_lease(lease_id, preparation).await } pub(super) async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) { @@ -205,6 +153,7 @@ impl HlsProxyManager { event: HlsLifecycleEvent, now_ms: u64, ) { + self.bind_lease_user_accounting(active_users); if !self.is_enabled() { return; } @@ -214,7 +163,8 @@ impl HlsProxyManager { let mut should_sync_session = false; if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await { should_sync_session = true; - if let Some(release) = &snapshot.idle_release { + let ending = matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied); + if let Some(release) = snapshot.idle_release.as_ref().filter(|_| !ending) { active_users .release_session_streams_and_counted_reservation( &release.username, @@ -228,7 +178,7 @@ impl HlsProxyManager { super::super::safe_user_session_token(&release.user_session_token) ); } - if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) { + if ending { self.remove_access_lease(&snapshot.lease_id).await; debug!( "HLS access lease removed: lease={} proxy_session={} state={}", @@ -385,6 +335,7 @@ impl HlsProxyManager { } pub fn exec_hls_lifecycle(ctx: &HlsCtx, cancel_token: &CancellationToken) { + ctx.attach_lease_user_accounting(); let hls_proxy = Arc::clone(&ctx.hls_proxy); let active_users = Arc::clone(&ctx.active_users); let active_provider = Arc::clone(&ctx.active_provider); diff --git a/backend/hls/src/manager/publication.rs b/backend/hls/src/manager/publication.rs index 8f6ac90c9..c5b5e407f 100644 --- a/backend/hls/src/manager/publication.rs +++ b/backend/hls/src/manager/publication.rs @@ -160,7 +160,7 @@ impl HlsProxyManager { gc.install_revision_store(&revision_store); segment_cache.install_revision_metrics(&metrics); let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); - Self { + let manager = Self { sessions, segment_cache, segment_repair, @@ -184,6 +184,7 @@ impl HlsProxyManager { transient_resources: Arc::new(TransientResourceStore::new()), access_leases, lifecycle, + lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()), account_overlap_cooldowns, metrics, qos, @@ -195,7 +196,9 @@ impl HlsProxyManager { availability_reevaluations, terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), startup_observability: Arc::new(HlsStartupObservability::default()), - } + }; + manager.install_worker_revision_recovery(); + manager } pub fn sessions(&self) -> &Arc { &self.sessions } @@ -475,6 +478,18 @@ impl HlsProxyManager { published_transient_resource_ids: HlsPublishedTransientResourceIds, now_ms: u64, ) -> HlsLeaseManifestPublicationOutcome { + if snapshot.startup_revisions.is_some() { + return self + .commit_recovered_startup_manifest( + lease_id, + proxy_session_id, + expected, + snapshot, + published_transient_resource_ids, + now_ms, + ) + .await; + } let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; let lease_issued_at_ms = expected.lease_issued_at_ms(); let Some(finalized_manifest_generation) = finalized_manifest_generation else { @@ -909,7 +924,11 @@ impl HlsProxyManager { } } self.schedule_session_idle_for_handle(&session).await; - session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); + session.live_window_strip = self.strip(); + } (session, outcome) } } diff --git a/backend/hls/src/manager/revision_recovery.rs b/backend/hls/src/manager/revision_recovery.rs new file mode 100644 index 000000000..13627b180 --- /dev/null +++ b/backend/hls/src/manager/revision_recovery.rs @@ -0,0 +1,247 @@ +use crate::{ + lease::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome}, + HlsAccessLeaseId, HlsMediaLeaseIdentity, HlsProxyManager, ProxySessionId, +}; +use std::sync::Arc; + +/// Manager-owned resources used by fetch completion, even when an origin I/O +/// context is absent. Sharing this handle avoids a second lease cleanup path. +#[derive(Clone)] +pub(crate) struct HlsLeaseRuntime { + pub(super) sessions: Arc, + pub(super) lease_completion: Arc, + pub(super) access_leases: Arc>, + pub(super) standalone_custom_access: Arc, + pub(super) segment_repair: Arc, + pub(super) startup_observability: Arc, + pub(super) qos: Arc, + pub(super) terminal_pending: Arc, + pub(super) terminal_commit_retries: Arc, +} + +impl HlsProxyManager { + pub(super) fn revision_recovery_runtime(&self) -> HlsLeaseRuntime { + HlsLeaseRuntime { + sessions: Arc::clone(&self.sessions), + lease_completion: Arc::clone(&self.lease_completion), + access_leases: Arc::clone(&self.access_leases), + standalone_custom_access: Arc::clone(&self.standalone_custom_access), + segment_repair: Arc::clone(&self.segment_repair), + startup_observability: Arc::clone(&self.startup_observability), + qos: Arc::clone(&self.qos), + terminal_pending: Arc::clone(&self.terminal_pending), + terminal_commit_retries: Arc::clone(&self.terminal_commit_retries), + } + } + + pub(super) fn install_worker_revision_recovery(&self) { + let _ = self.segment_worker_pool.revision_recovery.set(Arc::new(self.revision_recovery_runtime())); + } + + pub(super) async fn commit_recovered_startup_manifest( + &self, + lease_id: &HlsAccessLeaseId, + session_id: &ProxySessionId, + expected: crate::lease::HlsLeaseManifestPublicationGuard, + mut snapshot: crate::media_reserve::HlsLeaseManifestSnapshot, + resources: crate::HlsPublishedTransientResourceIds, + now_ms: u64, + ) -> crate::lease::HlsLeaseManifestPublicationOutcome { + use crate::lease::{ + HlsLeaseManifestPublicationOutcome as Outcome, HlsLeaseManifestPublicationRejectReason as Reject, + }; + let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { + return Outcome::Rejected(Reject::SessionMismatch); + }; + let mut leases = self.access_leases.write().await; + let mut session = current.session().write().await; + if !leases.media_identity_is_current(lease_id, session_id, expected.media_identity(), now_ms) + || session.is_gc_marked_for_removal() + { + return Outcome::Rejected(Reject::LeaseUnavailable); + } + if snapshot.startup_revisions.as_ref().is_some_and(|publication| publication.window_plan.is_some()) { + session.live_window_strip = self.strip(); + } + let retained = now_ms <= snapshot.delivered_at_ms.saturating_add(snapshot.playlist_duration_ms) + && leases + .response_snapshot(lease_id, session_id, now_ms) + .and_then(|lease| lease.last_manifest_snapshot) + .as_ref() + == Some(&snapshot); + if let Some(publication) = snapshot.startup_revisions.as_mut() { + if let Some(window) = &publication.window_plan { + let plan = &window.plan; + let view_matches = leases.response_snapshot(lease_id, session_id, now_ms).is_some_and(|lease| { + session.plan_live_window(crate::live_window::HlsLiveWindowView::for_lease(&lease)).as_ref() + == Some(plan) + }); + if (!plan.is_current(&session) || !view_matches) && !retained { + return Outcome::Rejected(Reject::ManifestGenerationUnavailable); + } + } + for (seq, revision) in &mut std::sync::Arc::make_mut(publication).revisions { + if let Some(guard) = leases.revision_recovery_guard(lease_id, *seq) { + match leases.recover_revision_binding(lease_id, &guard, &session, now_ms) { + HlsRevisionRecoveryOutcome::ReentryRequired => { + drop(session); + drop(leases); + drop(current); + self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await; + return Outcome::Rejected(Reject::LeaseUnavailable); + } + HlsRevisionRecoveryOutcome::RetryPending => return Outcome::Rejected(Reject::RevisionConflict), + HlsRevisionRecoveryOutcome::Unchanged | HlsRevisionRecoveryOutcome::ReplacementBound => {} + } + if let Some(bound) = leases.published_segment_revision(lease_id, *seq) { + if revision.revision().id() < bound.revision().id() + && matches!(revision.revision().state(), crate::SegmentRevisionState::Failed) + { + *revision = bound; + } + } + } + // Publication cannot introduce a revision left by a retired + // fetch, nor use a caller-supplied candidate outside startup. + if session.startup.as_ref().and_then(|startup| startup.revisions.get(seq)) != Some(revision) + && leases.published_segment_revision(lease_id, *seq).as_ref() != Some(revision) + { + return Outcome::Rejected(Reject::RevisionConflict); + } + } + } + let published = snapshot + .startup_revisions + .as_ref() + .and_then(|publication| publication.window_plan.as_ref()) + .map(|_| snapshot.clone()); + let outcome = leases + .commit_manifest_publication_with_resources(lease_id, session_id, expected, snapshot, resources, now_ms); + if outcome.is_committed() { + if let Some(snapshot) = published { + session.record_live_window_publication(lease_id, &snapshot); + } + } + outcome + } + + /// Resolves one published URI against the exact current session and lease. + /// The index -> lease store -> session order matches publication/cutover. + pub(crate) async fn recover_access_lease_revision( + &self, + lease_id: &HlsAccessLeaseId, + session_id: &ProxySessionId, + identity: HlsMediaLeaseIdentity, + seq: u64, + now_ms: u64, + ) -> HlsRevisionRecoveryOutcome { + let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { + return HlsRevisionRecoveryOutcome::Unchanged; + }; + let mut leases = self.access_leases.write().await; + let session = current.session().read().await; + if !leases.media_identity_is_current(lease_id, session_id, identity, now_ms) { + return HlsRevisionRecoveryOutcome::Unchanged; + } + let Some(guard) = leases.revision_recovery_guard(lease_id, seq) else { + return HlsRevisionRecoveryOutcome::Unchanged; + }; + let outcome = leases.recover_revision_binding(lease_id, &guard, &session, now_ms); + drop(session); + drop(leases); + drop(current); + if outcome == HlsRevisionRecoveryOutcome::ReentryRequired { + self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await; + } + outcome + } + + pub(crate) async fn end_access_lease_for_failed_revision( + &self, + lease_id: &HlsAccessLeaseId, + session_id: &ProxySessionId, + guard: &HlsRevisionRecoveryGuard, + now_ms: u64, + ) -> bool { + self.revision_recovery_runtime().end_access_lease_for_failed_revision(lease_id, session_id, guard, now_ms).await + } +} + +impl HlsLeaseRuntime { + /// Retire only the observed binding/snapshot/lease incarnation. Repeated + /// callers and completions from older snapshots are intentionally harmless. + pub(crate) async fn end_access_lease_for_failed_revision( + &self, + lease_id: &HlsAccessLeaseId, + session_id: &ProxySessionId, + guard: &HlsRevisionRecoveryGuard, + now_ms: u64, + ) -> bool { + let (issued_at_ms, snapshot_generation) = guard.completion_identity(); + let key = super::lease_completion::LeaseCompletionKey { + lease_id: lease_id.clone(), + session_id: session_id.clone(), + issued_at_ms, + snapshot_generation: Some(snapshot_generation), + }; + let runtime = self.clone(); + let lease_id = lease_id.clone(); + let session_id = session_id.clone(); + let guard = guard.clone(); + self.lease_completion + .run(key, async move { runtime.end_failed_revision_owned(&lease_id, &session_id, &guard, now_ms).await }) + .await + } + + async fn end_failed_revision_owned( + &self, + lease_id: &HlsAccessLeaseId, + session_id: &ProxySessionId, + guard: &HlsRevisionRecoveryGuard, + now_ms: u64, + ) -> bool { + let _users = self.lease_completion.users().await; + let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return false }; + let mut leases = self.access_leases.write().await; + let session = current.session().read().await; + let ended_at_ms = tuliprox_core::utils::current_time_millis().max(now_ms); + let preparation = leases.end_revision_playback_if_current(lease_id, guard, &session, ended_at_ms); + drop(session); + drop(leases); + drop(current); + let Some(preparation) = preparation else { return false }; + self.complete_prepared_lease(lease_id, &preparation).await + } + + pub(crate) async fn recover_segment_leases_after_fetch( + &self, + session_id: &ProxySessionId, + seq: u64, + origin_generation: u64, + now_ms: u64, + ) { + let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return }; + let mut leases = self.access_leases.write().await; + let session = current.session().read().await; + if session.activity.origin_work_generation != origin_generation || session.is_gc_marked_for_removal() { + return; + } + let ids = leases.revision_bound_lease_ids(session_id, seq); + let mut endings = Vec::new(); + for id in ids { + if let Some(guard) = leases.revision_recovery_guard(&id, seq) { + if leases.recover_revision_binding(&id, &guard, &session, now_ms) + == HlsRevisionRecoveryOutcome::ReentryRequired + { + endings.push((id, guard)); + } + } + } + drop(session); + drop(leases); + drop(current); + for (id, guard) in endings { + self.end_access_lease_for_failed_revision(&id, session_id, &guard, now_ms).await; + } + } +} diff --git a/backend/hls/src/manager/tests/behavior.rs b/backend/hls/src/manager/tests/behavior.rs index 7661e2514..2eed5eee0 100644 --- a/backend/hls/src/manager/tests/behavior.rs +++ b/backend/hls/src/manager/tests/behavior.rs @@ -194,6 +194,7 @@ async fn finalized_signed_url_generations_follow_published_lease_lifetime() { .await .expect("expired lease remains available for lifecycle cleanup"); assert_eq!(expired_a.state, HlsAccessLeaseState::Expired); + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); manager.remove_access_lease(&lease_a_id).await; let mut session = session.write().await; assert_eq!(session.transient.finalized_manifest_generation_count(), 1); @@ -309,6 +310,7 @@ async fn same_lease_retains_every_published_finalized_generation_until_removal() .is_some()); } + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); manager.remove_access_lease(&lease_id).await; let session = session.read().await; assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 0); @@ -434,6 +436,7 @@ async fn lease_reconciliation_cannot_resurrect_a_concurrently_removed_binding() ) }; wait_until_access_lease_store_is_write_locked(&manager).await; + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); let removal_task = { let manager = Arc::clone(&manager); let lease_id = lease_id.clone(); diff --git a/backend/hls/src/manager/tests/configuration.rs b/backend/hls/src/manager/tests/configuration.rs index 2588ed510..bd4769b7f 100644 --- a/backend/hls/src/manager/tests/configuration.rs +++ b/backend/hls/src/manager/tests/configuration.rs @@ -41,6 +41,7 @@ async fn update_config_applies_hls_runtime_settings_to_existing_manager() { assert_eq!(manager.strip().mode, HlsStripMode::Seconds); assert_eq!(manager.strip().value, 7); assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); + assert_eq!(session.read().await.live_window_strip, manager.strip()); } #[tokio::test] diff --git a/backend/hls/src/manager/tests/terminal.rs b/backend/hls/src/manager/tests/terminal.rs index d8f911359..ac6c31c84 100644 --- a/backend/hls/src/manager/tests/terminal.rs +++ b/backend/hls/src/manager/tests/terminal.rs @@ -1109,6 +1109,7 @@ async fn resource_denial_does_not_destroy_existing_terminal_tail() { ); assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); manager.remove_access_lease(&lease_id).await; assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); assert_eq!( @@ -1148,6 +1149,7 @@ async fn terminal_tail_protection_is_retained_until_bounded_lease_removal() { ); assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); manager.remove_access_lease(&lease_id).await; assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); } @@ -1188,6 +1190,7 @@ async fn terminal_unavailable_protection_is_retained_until_bounded_lease_removal assert_eq!(denied.state, HlsAccessLeaseState::Denied); assert!(matches!(denied.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users); manager.remove_access_lease(&lease_id).await; assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); } diff --git a/backend/hls/src/manifest_commit.rs b/backend/hls/src/manifest_commit.rs index efa0c8d7f..9cb072f4d 100644 --- a/backend/hls/src/manifest_commit.rs +++ b/backend/hls/src/manifest_commit.rs @@ -112,6 +112,35 @@ pub fn hls_committed_manifest_body_for_request( }) } +/// The accelerated lease view is rendered from an origin commit, independently +/// of whether the shared session's older window can still be rendered. +pub fn hls_live_window_origin_allowed_for_request( + session: &HlsSession, + options: HlsCachedManifestOptions, + started_at_ms: u64, + now_ms: u64, +) -> bool { + let Some(identity) = session.live_window_source_identity() else { return false }; + let duration = session.origin_window_durations_ms.iter().fold(0_u64, |sum, duration| sum.saturating_add(*duration)); + let candidate = HlsCommittedManifestCandidate { + body: HlsCommittedManifestBody::Normal(String::new()), + rendered_at_ms: identity.rendered_at_ms(), + valid_until_ms: Some(identity.rendered_at_ms().saturating_add(duration)), + finalized: false, + }; + session.uses_fast_start_live_window() + && can_serve_committed_manifest( + session, + &candidate, + session.account_binding_protection(now_ms), + options.policy, + candidate.rendered_at_ms >= started_at_ms, + now_ms, + ) + && manifest_rendered_after_required_boundary(Some(candidate.rendered_at_ms), options) + && options.after_origin_manifest_commits.is_none_or(|commits| session.origin_manifest_commits() > commits) +} + pub fn hls_should_wait_for_initial_manifest_commit( session: &HlsSession, selected_committed_body: bool, diff --git a/backend/hls/src/manifest_snapshot.rs b/backend/hls/src/manifest_snapshot.rs index de7015f94..bb485fcb1 100644 --- a/backend/hls/src/manifest_snapshot.rs +++ b/backend/hls/src/manifest_snapshot.rs @@ -15,6 +15,12 @@ use std::sync::Arc; /// Exact manifest delivery from which lease-local playback evidence is derived. pub enum HlsLeaseManifestSnapshotInput<'a> { + FastStartWindow { + session: &'a HlsSession, + plan: &'a crate::live_window::HlsLiveWindowPlan, + rendered: &'a crate::renderer::RenderedManifest, + materialized_body: &'a str, + }, NormalCacheTimeline { session: &'a HlsSession, committed_body: &'a str, @@ -91,6 +97,12 @@ pub fn derive_hls_lease_manifest_snapshot( delivered_at_ms: u64, ) -> Result, HlsManifestLimitViolation> { match input { + HlsLeaseManifestSnapshotInput::FastStartWindow { session, plan, rendered, materialized_body } => { + if !plan.is_current(session) || rendered.segment_proxy_seqs != plan.visible { + return Ok(None); + } + derive_rendered_snapshot(session, rendered, materialized_body, 0, delivered_at_ms, Some((*plan).clone())) + } HlsLeaseManifestSnapshotInput::NormalCacheTimeline { session, committed_body, @@ -155,7 +167,22 @@ fn derive_normal_snapshot( if rendered.body != committed_body { return Ok(None); } - let Some(source_commit_identity) = session.last_normal_manifest_commit_identity() else { + derive_rendered_snapshot(session, rendered, materialized_body, stripped_tail_segments, delivered_at_ms, None) +} + +fn derive_rendered_snapshot( + session: &HlsSession, + rendered: &crate::renderer::RenderedManifest, + materialized_body: &str, + stripped_tail_segments: usize, + delivered_at_ms: u64, + window_plan: Option, +) -> Result, HlsManifestLimitViolation> { + let Some(source_commit_identity) = window_plan + .as_ref() + .and_then(|plan| plan.source_commit_identity) + .or_else(|| session.last_normal_manifest_commit_identity()) + else { return Ok(None); }; let Some(parsed) = parse_lease_manifest(materialized_body)? else { @@ -210,6 +237,10 @@ fn derive_normal_snapshot( .map(|seq| Some((*seq, startup.revisions.get(seq)?.clone()))) .collect::>>()?; snapshot.startup_revisions = Some(Arc::new(super::HlsManifestRevisions { + window_plan: window_plan.map(|plan| crate::live_window::HlsLiveWindowPublication { + plan, + body: Arc::from(materialized_body), + }), mode: startup.config.mode, retained_start_seq: session.segments.first_key_value().map_or(first_proxy_seq, |(seq, _)| *seq), revisions, diff --git a/backend/hls/src/media_reserve.rs b/backend/hls/src/media_reserve.rs index cd482f9a5..d82b3e423 100644 --- a/backend/hls/src/media_reserve.rs +++ b/backend/hls/src/media_reserve.rs @@ -76,7 +76,6 @@ impl HlsManifestCommitIdentity { pub const fn commit_generation(self) -> u64 { self.commit_generation } - #[cfg(any(test, feature = "test-support"))] pub const fn rendered_at_ms(self) -> u64 { self.rendered_at_ms } } @@ -1010,6 +1009,7 @@ mod tests { ); } manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::FirstReady, retained_start_seq: 0, revisions: owners, @@ -1055,6 +1055,7 @@ mod tests { ); } manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Progressive, retained_start_seq: 0, revisions: owners, @@ -1092,6 +1093,7 @@ mod tests { ); } manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: shared::model::HlsStartupMode::Optimistic, retained_start_seq: 0, revisions: owners, diff --git a/backend/hls/src/prefetch.rs b/backend/hls/src/prefetch.rs index c0f706990..2f9f99e22 100644 --- a/backend/hls/src/prefetch.rs +++ b/backend/hls/src/prefetch.rs @@ -204,6 +204,50 @@ impl HlsSession { if self.is_gc_marked_for_removal() { return report; } + if self.uses_fast_start_live_window() { + let plans = self.required_live_windows(); + let required = plans.iter().flat_map(|plan| plan.visible.iter().copied()).collect::>(); + let obsolete = self + .segment_prefetch_queue + .proxy_seqs() + .into_iter() + .filter(|seq| { + !required.contains(seq) + && self.segments.get(seq).is_some_and(|entry| { + matches!( + entry.status, + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. } + ) + }) + }) + .collect::>(); + for seq in obsolete { + self.segment_prefetch_queue.remove(seq); + if let Some(entry) = self.segments.get_mut(&seq) { + entry.status = SegmentCacheStatus::Discovered; + } + } + for seq in required { + if self.queue_segment_fetch_candidate(seq, SegmentFetchPriority::RenderWindow, now_ms) { + report.render_window_queued += 1; + } + } + let reserve = plans.iter().flat_map(|plan| plan.hidden_tail.iter().copied()).collect::>(); + for seq in reserve { + if allow_prefetch { + if self.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Prefetch, now_ms) { + report.prefetch_queued += 1; + } + } else if self + .segments + .get(&seq) + .is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Discovered)) + { + report.prefetch_skipped += 1; + } + } + return report; + } let Some(tail_proxy_seq) = self.publishable_origin_tail_proxy_seq else { return report; }; diff --git a/backend/hls/src/refresh/commit.rs b/backend/hls/src/refresh/commit.rs index 8b3c60d5e..180e6b1d2 100644 --- a/backend/hls/src/refresh/commit.rs +++ b/backend/hls/src/refresh/commit.rs @@ -895,6 +895,7 @@ fn commit_normal_manifest( session.record_origin_manifest_commit(); session.transient.upsert_resources(key_resources); session.initial_prefetch_gap_segments = initial_prefetch_gap_segments; + session.record_live_window_origin_commit(rendered_at_ms); if provisioning_handoff { limit_publishable_normal_provisioning_handoff_tail(session, &request.strip, manifest.segments.len()); } diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index 947b13d59..ffadeee85 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -92,3 +92,116 @@ use self::support::{ StaleSwitchGeneration, TestOriginServer, CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, SWITCH_MAP_BODY, SWITCH_SEGMENT_BODY, }; + +#[tokio::test] +#[allow( + clippy::too_many_lines, + reason = "Keeps the degraded admission and unchanged-success counterprobe in one scenario." +)] +async fn retryable_manifest_failure_keeps_optimistic_admission_degraded_until_media_progress() -> std::io::Result<()> { + use crate::{ + media_reserve::{ + evaluate_startup_admission, HlsReadyTimelineSnapshot, HlsStartupAdmissionDecision, + HlsStartupAdmissionInput, HlsStartupAdmissionOriginState, + }, + origin_progress::HlsOriginPathCondition, + }; + use std::sync::Arc; + let session = test_session(); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: Arc::clone(&session), + availability_runtime: None, + origin_entry: LiveHlsOriginEntry::parse("http://127.0.0.1:9/index.m3u8").unwrap(), + headers: axum::http::HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_manifest_timeout_ms: 1000, + manifest_recovery_burst: tuliprox_core::model::HlsManifestRecoveryBurstConfig::default(), + retry_policy: RetryPolicy::default(), + recovery_timing_policy: test_recovery_timing_policy(100), + acceptance_timing_seed: None, + }; + let manifest = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4,\n0.ts\n#EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n"); + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(origin) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + &manifest.body, + &manifest.final_manifest_url, + ) + else { + panic!("normal"); + }; + let snapshot = { + let mut session = session.write().await; + session.apply_origin_manifest(&origin).unwrap(); + session.origin_control.record_media_progress(10, 4000); + let config = tuliprox_core::model::HlsStartupConfig { + mode: shared::model::HlsStartupMode::Optimistic, + ..Default::default() + }; + let store = Arc::new(crate::SegmentRevisionStore::default()); + let mut revisions = std::collections::BTreeMap::new(); + for seq in 0..3 { + revisions + .insert(seq, store.create(session.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?); + } + revisions[&0].revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + session.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 10000, + worker: std::sync::Weak::new(), + access_leases: Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())), + revisions, + policy_fixed: true, + store, + budget: crate::ProgressiveBudgetManager::new(config), + }); + let rendered = session.render_and_store_manifest(10).unwrap(); + crate::api::derive_hls_lease_manifest_snapshot( + &crate::api::HlsLeaseManifestSnapshotInput::NormalCacheTimeline { + session: &session, + committed_body: &rendered.body, + materialized_body: &rendered.body, + stripped_tail_segments: 0, + }, + 10, + ) + .unwrap() + .unwrap() + }; + let evaluate = |origin_state| { + evaluate_startup_admission(HlsStartupAdmissionInput { + manifest: &snapshot, + ready_timeline: &HlsReadyTimelineSnapshot { units: Arc::from([]) }, + origin_state, + recovery_trigger_budget: crate::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0), + }) + .decision + }; + assert_eq!(evaluate(HlsStartupAdmissionOriginState::Healthy), HlsStartupAdmissionDecision::Admit); + let generation = session.read().await.origin_control.progress_generation; + crate::availability::record_retryable_manifest_attempt_failure( + &context, + generation, + &OriginManifestFetchError::RetryableStatus(axum::http::StatusCode::BAD_GATEWAY, None), + 11, + ) + .await; + assert_eq!(session.read().await.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure); + assert!(matches!(evaluate(HlsStartupAdmissionOriginState::Degraded), HlsStartupAdmissionDecision::Reject(_))); + { + let mut session = session.write().await; + let timing = build_manifest_refresh_timing(Some(4000), Some(4000), HlsManifestProgress::Unchanged); + record_committed_manifest_media_progress( + &mut session.origin_control, + HlsManifestCommitProgressEvidence::CacheTimeline(timing), + Some(4), + 12, + ); + assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure); + session.origin_control.record_media_progress(13, 4000); + assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::ProgressExpected); + } + Ok(()) +} diff --git a/backend/hls/src/renderer.rs b/backend/hls/src/renderer.rs index 8ae716d64..23a1a0aa5 100644 --- a/backend/hls/src/renderer.rs +++ b/backend/hls/src/renderer.rs @@ -91,7 +91,43 @@ enum RenderTailPolicy { } impl HlsManifestRenderer { + pub fn render_live_window( + session: &HlsSession, + plan: &crate::live_window::HlsLiveWindowPlan, + rendered_at_ms: u64, + ) -> Result { + if !plan.is_current(session) { + return Err(RenderError::InvalidState); + } + let minimum = if session.allows_short_origin_windows() { 1 } else { MIN_VISIBLE_SEGMENTS }; + let startup = session.startup.as_ref().ok_or(RenderError::InvalidState)?; + let head = plan.visible.first().ok_or(RenderError::NoRenderableWindow)?; + if plan.visible.len() < minimum + || !startup + .revisions + .get(head) + .is_some_and(|revision| revision.revision().is_startup_head_ready(startup.config.mode)) + || !plan + .visible + .iter() + .all(|seq| session.segments.get(seq).is_some_and(|entry| is_renderable(entry, session))) + { + return Err(RenderError::NoRenderableWindow); + } + render_window(session, &plan.visible, plan.effective_strip_segments, rendered_at_ms) + } + pub fn render(session: &HlsSession, rendered_at_ms: u64) -> Result { + if session.uses_fast_start_live_window() { + let view = session + .last_rendered_manifest + .as_ref() + .map_or(crate::live_window::HlsLiveWindowView::Entry, |previous| { + crate::live_window::HlsLiveWindowView::Active { first_published: previous.first_proxy_seq } + }); + let plan = session.plan_live_window(view).ok_or(RenderError::NoRenderableWindow)?; + return Self::render_live_window(session, &plan, rendered_at_ms); + } for render_gap_segments in 0..=session.render_policy.initial_render_gap_segments { let Some(window) = select_window(session, render_gap_segments, RenderTailPolicy::TruncateUnavailableTail) else { @@ -196,6 +232,14 @@ impl HlsSession { } pub fn renderer_candidate_window_proxy_seqs(session: &HlsSession) -> Vec { + if session.uses_fast_start_live_window() { + return session + .required_live_windows() + .into_iter() + .flat_map(|plan| plan.visible) + .chain(session.live_window_demands.values().flat_map(|demand| demand.published.iter().copied())) + .collect(); + } if session.startup.is_some() { let Some((head, tail)) = session.publishable_origin_head_proxy_seq.zip(session.publishable_origin_tail_proxy_seq) @@ -352,6 +396,14 @@ fn retain_published_predecessors(session: &HlsSession, window: &mut Vec) { else { return; }; + retain_published_predecessors_from(session, window, previous_first_proxy_seq); +} + +pub(crate) fn retain_published_predecessors_from( + session: &HlsSession, + window: &mut Vec, + previous_first_proxy_seq: u64, +) { let Some(&first_proxy_seq) = window.first() else { return; }; diff --git a/backend/hls/src/response.rs b/backend/hls/src/response.rs index f56dd0c3d..7dfabc7e6 100644 --- a/backend/hls/src/response.rs +++ b/backend/hls/src/response.rs @@ -84,6 +84,8 @@ pub struct HlsCacheResponseContext { pub segment_repair: Arc, pub qos_meter: Arc>, pub media_activity_marker: Option, + /// Only GET may reserve a lease's progressive opportunity; HEAD waits for metadata. + pub request_method: axum::http::Method, pub now_ms: u64, } diff --git a/backend/hls/src/response/cache.rs b/backend/hls/src/response/cache.rs index 916a4d176..6f5e20ef0 100644 --- a/backend/hls/src/response/cache.rs +++ b/backend/hls/src/response/cache.rs @@ -69,6 +69,7 @@ impl HlsCacheResponseContext { segment_repair, qos_meter: Arc::new(ArcSwapOption::from(qos_meter)), media_activity_marker, + request_method: axum::http::Method::GET, now_ms, } } diff --git a/backend/hls/src/response/revision.rs b/backend/hls/src/response/revision.rs index 5ceb3a733..619903999 100644 --- a/backend/hls/src/response/revision.rs +++ b/backend/hls/src/response/revision.rs @@ -71,24 +71,16 @@ async fn prepare_hls_revision_response( (Duration::from_secs(startup.config.max_progressive_reader_lifetime_secs.get()), startup.first_data_timeout_ms) }; let deadline = deadline.min(remaining_wait_deadline(context.now_ms, wait_ms)); - let (revision, progressive) = { - let mut leases = marker.manager.access_leases().write().await; - let Some(revision) = leases.published_segment_revision(&context.hls_access_lease_id, segment_file.proxy_seq) - else { - return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing); - }; - let full_request = range.is_none_or(|range| range.as_bytes() == b"bytes=0-"); - let progressive = revision.revision().key.kind == super::super::SegmentRevisionKind::Raw - && full_request - && leases.try_claim_progressive_startup( - &context.hls_access_lease_id, - segment_file.proxy_seq, - context.now_ms, - ); - (revision, progressive) - }; - let metadata = match revision_metadata(revision.revision(), progressive, deadline).await { - Ok(metadata) => metadata, + let (revision, metadata) = match resolve_revision_metadata( + marker, + segment_file, + range, + context.request_method == axum::http::Method::GET, + deadline, + ) + .await + { + Ok(resolved) => resolved, Err(failure) => return HlsResourceServeOutcome::Failure(failure), }; let authorized_at_ms = current_time_millis(); @@ -154,6 +146,95 @@ fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::I tokio::time::Instant::now() + Duration::from_millis(wait_ms.saturating_sub(elapsed_ms)) } +async fn resolve_revision_metadata( + marker: &super::HlsMediaActivityMarker, + segment_file: &HlsSegmentFile, + range: Option<&HeaderValue>, + is_get: bool, + deadline: tokio::time::Instant, +) -> Result<(crate::SegmentRevisionGuard, Option), HlsResourceServeFailure> { + use crate::lease::HlsRevisionRecoveryOutcome; + // Reserve before waiting for recovery or the first body bytes. A concurrent + // request must not steal the opportunity from a headers-only first request. + let startup_claimed = { + let mut leases = marker.manager.access_leases().write().await; + let now_ms = current_time_millis(); + if !leases.media_identity_is_current(&marker.lease_id, &marker.proxy_session_id, marker.lease_identity, now_ms) + { + return Err(HlsResourceServeFailure::Expired); + } + if leases.published_segment_revision(&marker.lease_id, segment_file.proxy_seq).is_none() { + return Err(HlsResourceServeFailure::Missing); + } + let full_request = is_get && range.is_none_or(|range| range.as_bytes() == b"bytes=0-"); + full_request && leases.try_claim_progressive_startup(&marker.lease_id, segment_file.proxy_seq, now_ms) + }; + let mut last_outcome = None; + loop { + if tokio::time::Instant::now() >= deadline { + return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }); + } + let now_ms = current_time_millis(); + let outcome = marker + .manager + .recover_access_lease_revision( + &marker.lease_id, + &marker.proxy_session_id, + marker.lease_identity, + segment_file.proxy_seq, + now_ms, + ) + .await; + if outcome == HlsRevisionRecoveryOutcome::ReentryRequired { + return Err(HlsResourceServeFailure::Missing); + } + let (revision, progressive) = { + let mut leases = marker.manager.access_leases().write().await; + if !leases.media_identity_is_current( + &marker.lease_id, + &marker.proxy_session_id, + marker.lease_identity, + now_ms, + ) { + return Err(HlsResourceServeFailure::Expired); + } + let revision = leases + .published_segment_revision(&marker.lease_id, segment_file.proxy_seq) + .ok_or(HlsResourceServeFailure::Missing)?; + if outcome == HlsRevisionRecoveryOutcome::RetryPending { + if last_outcome != Some(outcome) { + let session = marker.session.read().await; + log::debug!( + "HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=retry_pending", + crate::safe_hls_access_lease_id(&marker.lease_id), segment_file.proxy_seq, + session.activity.origin_work_generation, revision.revision().id(), + revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire), + revision.revision().was_exposed(), + ); + } + last_outcome = Some(outcome); + drop(leases); + tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await; + continue; + } + let progressive = startup_claimed && revision.revision().key.kind == crate::SegmentRevisionKind::Raw; + (revision, progressive) + }; + match revision_metadata(revision.revision(), progressive, deadline).await { + Ok(metadata) => return Ok((revision, metadata)), + Err(HlsResourceServeFailure::PermanentFailed { .. }) => { + // A pending revision may fail while this request waits for EOF. + // Re-resolve through the same fenced transaction and deadline. + if !matches!(revision.revision().state(), SegmentRevisionState::Failed) { + return Err(HlsResourceServeFailure::PermanentFailed { status: None }); + } + tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await; + } + Err(failure) => return Err(failure), + } + } +} + /// A progressive claim streams a publishable pending revision; every other /// request waits for the complete revision file. async fn revision_metadata( @@ -162,11 +243,26 @@ async fn revision_metadata( deadline: tokio::time::Instant, ) -> Result, HlsResourceServeFailure> { if progressive { - return match revision.state() { - SegmentRevisionState::Complete(metadata) => Ok(Some(metadata)), - SegmentRevisionState::Pending if revision.is_publishable() => Ok(None), - _ => Err(HlsResourceServeFailure::PermanentFailed { status: None }), - }; + let mut changes = revision.subscribe(); + loop { + let state = changes.borrow_and_update().clone(); + match state { + SegmentRevisionState::Complete(metadata) => return Ok(Some(metadata)), + SegmentRevisionState::Failed => { + return Err(HlsResourceServeFailure::PermanentFailed { status: None }); + } + SegmentRevisionState::Pending if revision.can_expose() => return Ok(None), + SegmentRevisionState::Pending => {} + } + // Optimistic successors can be listed before any body byte arrives. + // Without replay (or after replay exhaustion) wait for the complete + // immutable file instead; never expose a missing prefix as a 404. + match tokio::time::timeout_at(deadline, changes.changed()).await { + Ok(Ok(())) => {} + Ok(Err(_)) => return Err(HlsResourceServeFailure::PermanentFailed { status: None }), + Err(_) => return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }), + } + } } match revision.wait_complete(deadline).await { Ok(metadata) => Ok(Some(metadata)), diff --git a/backend/hls/src/response/tests.rs b/backend/hls/src/response/tests.rs index 38a23ec1a..1b2e7150c 100644 --- a/backend/hls/src/response/tests.rs +++ b/backend/hls/src/response/tests.rs @@ -67,6 +67,7 @@ async fn live_media_marker( struct CachedSegmentFixture { _temp_dir: tempfile::TempDir, manager: Arc, + active_users: Arc, segment_cache: Arc, session: HlsSessionHandle, proxy_session_id: ProxySessionId, @@ -80,8 +81,15 @@ struct CachedSegmentFixture { impl CachedSegmentFixture { async fn new(bytes: Vec) -> Self { + let fixture = Self::without_lease_user_accounting(bytes).await; + fixture.manager.bind_lease_user_accounting(&fixture.active_users); + fixture + } + + async fn without_lease_user_accounting(bytes: Vec) -> Self { let temp_dir = tempfile::tempdir().expect("tempdir"); let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let active_users = crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users; let segment_cache = Arc::clone(manager.segment_cache()); let session = manager.get_or_create_session(HlsSessionKey::new(1, "range-stream"), b"secret", 1_000).await; let proxy_session_id = session.read().await.proxy_session_id.clone(); @@ -137,6 +145,7 @@ impl CachedSegmentFixture { Self { _temp_dir: temp_dir, manager, + active_users, segment_cache, session, proxy_session_id, @@ -202,6 +211,7 @@ async fn publish_revision_fixture( fixture.segment_file.proxy_seq, if raw { crate::SegmentRevisionKind::Raw } else { crate::SegmentRevisionKind::Processed }, )?; + revision.revision().bind_origin(&*fixture.session.read().await); if raw { let mut replay = crate::progressive_startup::ProgressiveReplay::new(&budget).ok_or_else(|| std::io::Error::other("slot"))?; @@ -243,6 +253,7 @@ async fn publish_revision_fixture( }); let snapshot = crate::HlsLeaseManifestSnapshot { startup_revisions: Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: config.mode, retained_start_seq: fixture.segment_file.proxy_seq, revisions, @@ -283,8 +294,10 @@ async fn publish_revision_fixture( } mod admission; +mod first_requested; mod http; mod lifecycle; mod policy; mod recovery; +mod revision_recovery; mod terminal; diff --git a/backend/hls/src/response/tests/first_requested.rs b/backend/hls/src/response/tests/first_requested.rs new file mode 100644 index 000000000..96eda42f2 --- /dev/null +++ b/backend/hls/src/response/tests/first_requested.rs @@ -0,0 +1,309 @@ +//! Startup follows the player's first full GET, including a headers-only successor. +use super::*; +use crate::sync_ext::MutexExt; + +const BODY: &[u8] = b"prefix-tail"; + +struct FirstRequestFixture { + cached: CachedSegmentFixture, + revisions: std::collections::BTreeMap, +} + +impl FirstRequestFixture { + async fn new(mode: shared::model::HlsStartupMode, with_prefix: bool, with_replay: bool) -> std::io::Result { + let cached = CachedSegmentFixture::new(BODY.to_vec()).await; + let head = publish_revision_fixture(&cached, true).await?; + let mut revisions = std::collections::BTreeMap::from([(12, head)]); + { + let mut session = cached.session.write().await; + session.startup.as_mut().expect("startup").config.mode = mode; + let store = Arc::clone(&session.startup.as_ref().expect("startup").store); + let budget = Arc::clone(&session.startup.as_ref().expect("startup").budget); + for seq in [13, 14] { + let mut entry = session.segments[&12].clone(); + entry.proxy_seq = seq; + entry.origin_key.host_local_sequence = seq; + entry.origin_key.host_local_index = u32::try_from(seq).expect("fixture sequence"); + entry.cache_key = SegmentCacheKey::new(cached.proxy_session_id.clone(), seq, "ts"); + entry.media_start_ms = seq * 4000; + entry.status = SegmentCacheStatus::Fetching { + priority: crate::SegmentFetchPriority::Demand, + started_at_ms: current_time_millis(), + }; + session.segments.insert(seq, entry); + let revision = store.create(cached.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?; + revision.revision().bind_origin(&session); + if with_replay { + let mut replay = crate::progressive_startup::ProgressiveReplay::new(&budget) + .ok_or_else(|| std::io::Error::other("replay slot"))?; + if with_prefix { + replay.push(b"prefix")?; + revision.revision().prefix_available.store(6, Ordering::Release); + } + *revision.revision().replay.lock_unpoisoned() = Some(replay); + } + session.startup.as_mut().expect("startup").revisions.insert(seq, revision.clone()); + revisions.insert(seq, revision); + } + } + let mut leases = cached.manager.access_leases().write().await; + let mut snapshot = leases + .response_snapshot(&cached.lease_id, &cached.proxy_session_id, current_time_millis()) + .expect("lease") + .last_manifest_snapshot + .expect("snapshot"); + let mut segments = snapshot.visible_segments.to_vec(); + for seq in [13, 14] { + let mut segment = segments[0].clone(); + segment.proxy_seq = seq; + segment.uri = format!("{seq}.ts").into(); + segments.push(segment); + } + snapshot.visible_segments = Arc::from(segments); + snapshot.last_proxy_seq = 14; + snapshot.playlist_duration_ms = 12_000; + snapshot.last_visible_media_end_ms = 12_000; + snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, + mode, + retained_start_seq: 12, + revisions: revisions.clone(), + })); + let now = current_time_millis(); + let guard = + leases.prepare_manifest_publication(&cached.lease_id, &cached.proxy_session_id, now).expect("guard"); + assert!(leases + .commit_manifest_publication(&cached.lease_id, &cached.proxy_session_id, guard, snapshot, now) + .is_committed()); + drop(leases); + Ok(Self { cached, revisions }) + } + + async fn serve(&self, seq: u64, range: Option<&str>) -> HlsResourceServeOutcome { + self.serve_method(seq, range, axum::http::Method::GET).await + } + + async fn serve_method(&self, seq: u64, range: Option<&str>, method: axum::http::Method) -> HlsResourceServeOutcome { + let context = HlsCacheResponseContext { + hls_access_lease_id: self.cached.lease_id.clone(), + log_identity: self.cached.context.log_identity.clone(), + cache_duration_seconds: self.cached.context.cache_duration_seconds, + metrics: Arc::clone(&self.cached.context.metrics), + segment_repair: Arc::clone(&self.cached.context.segment_repair), + qos_meter: Arc::clone(&self.cached.context.qos_meter), + media_activity_marker: self.cached.context.media_activity_marker.clone(), + request_method: method, + now_ms: current_time_millis(), + }; + serve_hls_segment_cache_outcome( + Arc::clone(&self.cached.segment_cache), + Arc::clone(&self.cached.session), + HlsSegmentFile { proxy_seq: seq, extension: "ts".into() }, + range.map(header), + &context, + ) + .await + } + + fn prefix(&self, seq: u64) -> std::io::Result<()> { + let revision = self.revisions[&seq].revision(); + revision.replay.lock_unpoisoned().as_mut().expect("replay").push(b"prefix")?; + revision.prefix_available.store(6, Ordering::Release); + revision.prefix_changed(); + Ok(()) + } + + async fn complete(&self, seq: u64) -> std::io::Result<()> { + let cache_key = self.cached.session.read().await.segments[&seq].cache_key.clone(); + let metadata = self.cached.segment_cache.write_bytes_and_commit(&cache_key, BODY).await?; + assert!(self.revisions[&seq].revision().complete(metadata)); + self.revisions[&seq].revision().replay.lock_unpoisoned().take(); + self.cached.session.write().await.segments.get_mut(&seq).expect("entry").status = SegmentCacheStatus::Ready { + content_length: u64::try_from(BODY.len()).expect("body length"), + ready_at_ms: current_time_millis(), + }; + Ok(()) + } +} + +#[tokio::test] +async fn first_successor_streams_before_eof_in_both_raw_startup_modes() -> std::io::Result<()> { + for mode in [shared::model::HlsStartupMode::Progressive, shared::model::HlsStartupMode::Optimistic] { + let fixture = FirstRequestFixture::new(mode, true, true).await?; + let response = ready_response(fixture.serve(13, None).await); + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + let mut body = response.into_body().into_data_stream(); + assert_eq!( + body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), + Some(b"prefix".as_slice()) + ); + assert!(fixture.revisions[&13].revision().was_exposed()); + assert!(!fixture.revisions[&12].revision().was_exposed()); + assert!(!fixture.revisions[&13].revision().is_complete()); + let subsequent = fixture.serve(14, None); + tokio::pin!(subsequent); + assert!(futures::poll!(subsequent.as_mut()).is_pending()); + fixture.complete(14).await?; + let subsequent = ready_response(subsequent.await); + assert!(subsequent.headers().contains_key(header::CONTENT_LENGTH)); + assert_eq!(subsequent.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY); + fixture.complete(13).await?; + assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"-tail".as_slice())); + assert!(body.next().await.is_none()); + } + Ok(()) +} + +#[tokio::test] +async fn headers_only_first_get_keeps_the_claim_until_its_own_prefix_arrives() -> std::io::Result<()> { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?; + let first = fixture.serve(13, Some("bytes=0-")); + let later = fixture.serve(14, None); + tokio::pin!(first, later); + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(later.as_mut()).is_pending()); + fixture.prefix(14)?; + assert!(futures::poll!(later.as_mut()).is_pending(), "a later GET still waits for EOF"); + fixture.prefix(13)?; + let response = ready_response(first.await); + assert_eq!(response.status(), StatusCode::OK); + let mut body = response.into_body().into_data_stream(); + assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice())); + assert!(futures::poll!(later.as_mut()).is_pending()); + fixture.complete(14).await?; + assert!(ready_response(later.await).headers().contains_key(header::CONTENT_LENGTH)); + Ok(()) +} + +#[tokio::test] +async fn headers_only_first_get_times_out_temporarily_and_preserves_the_lease() -> std::io::Result<()> { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?; + fixture.cached.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25; + let failure = match fixture.serve(13, None).await { + HlsResourceServeOutcome::Failure(failure) => failure, + HlsResourceServeOutcome::Ready(_) => panic!("the origin has not supplied a prefix"), + }; + let response = crate::response::cache::hls_resource_failure_default_response(failure); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(header::RETRY_AFTER)); + assert!(fixture + .cached + .manager + .access_lease_response_snapshot( + &fixture.cached.lease_id, + &fixture.cached.proxy_session_id, + current_time_millis() + ) + .await + .is_some()); + fixture.complete(13).await?; + let retry = ready_response(fixture.serve(13, None).await); + assert!(retry.headers().contains_key(header::CONTENT_LENGTH)); + assert_eq!(retry.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY); + Ok(()) +} + +#[tokio::test] +async fn head_metadata_probe_does_not_consume_the_first_get_opportunity() -> std::io::Result<()> { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?; + let head = fixture.serve_method(12, None, axum::http::Method::HEAD); + tokio::pin!(head); + assert!(futures::poll!(head.as_mut()).is_pending(), "HEAD waits for complete metadata"); + let first_get = ready_response(fixture.serve(13, None).await); + assert!(!first_get.headers().contains_key(header::CONTENT_LENGTH)); + assert!(fixture.revisions[&13].revision().was_exposed()); + assert!(!fixture.revisions[&12].revision().was_exposed()); + fixture.complete(12).await?; + assert!(ready_response(head.await).headers().contains_key(header::CONTENT_LENGTH)); + Ok(()) +} + +#[tokio::test] +async fn first_get_without_replay_or_after_unexposed_replay_exhaustion_waits_for_complete_bytes() -> std::io::Result<()> +{ + for replay in [false, true] { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, replay).await?; + let first = fixture.serve(13, None); + tokio::pin!(first); + assert!(futures::poll!(first.as_mut()).is_pending()); + if replay { + fixture.revisions[&13].revision().replay_exhausted(); + assert!(futures::poll!(first.as_mut()).is_pending()); + } + fixture.complete(13).await?; + let response = ready_response(first.await); + assert!(response.headers().contains_key(header::CONTENT_LENGTH)); + assert_eq!(response.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY); + assert!(!fixture.revisions[&13].revision().was_exposed()); + } + Ok(()) +} + +#[tokio::test] +async fn completed_first_get_consumes_startup_but_a_partial_range_does_not() -> std::io::Result<()> { + for partial in [false, true] { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?; + fixture.complete(12).await?; + let first = ready_response(fixture.serve(12, if partial { Some("bytes=2-5") } else { None }).await); + assert_eq!(first.status(), if partial { StatusCode::PARTIAL_CONTENT } else { StatusCode::OK }); + first.into_body().collect().await.map_err(std::io::Error::other)?; + let next = fixture.serve(13, None); + tokio::pin!(next); + if partial { + let response = ready_response(next.await); + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + } else { + assert!(futures::poll!(next.as_mut()).is_pending()); + fixture.complete(13).await?; + assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH)); + } + } + Ok(()) +} + +#[tokio::test] +async fn missing_uri_does_not_consume_startup_and_cancelled_first_get_does() -> std::io::Result<()> { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?; + assert!(matches!( + fixture.serve(99, None).await, + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing) + )); + { + let first = fixture.serve(13, None); + tokio::pin!(first); + assert!(futures::poll!(first.as_mut()).is_pending()); + } + fixture.prefix(14)?; + let next = fixture.serve(14, None); + tokio::pin!(next); + assert!(futures::poll!(next.as_mut()).is_pending()); + fixture.complete(14).await?; + assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH)); + Ok(()) +} + +#[tokio::test] +async fn first_successor_body_failure_is_terminal_for_the_exposed_revision() -> std::io::Result<()> { + let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?; + let response = ready_response(fixture.serve(13, None).await); + let mut body = response.into_body().into_data_stream(); + assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice())); + fixture.revisions[&13].revision().fail(); + assert!(body.next().await.is_some_and(|chunk| chunk.is_err())); + assert!(body.next().await.is_none()); + assert!(matches!( + fixture.serve(13, None).await, + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing) + )); + assert!(fixture + .cached + .manager + .access_lease_response_snapshot( + &fixture.cached.lease_id, + &fixture.cached.proxy_session_id, + current_time_millis() + ) + .await + .is_none()); + Ok(()) +} diff --git a/backend/hls/src/response/tests/lifecycle.rs b/backend/hls/src/response/tests/lifecycle.rs index 5d5083d1c..b300c5421 100644 --- a/backend/hls/src/response/tests/lifecycle.rs +++ b/backend/hls/src/response/tests/lifecycle.rs @@ -131,7 +131,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() { } #[tokio::test] -async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segment_failure() -> std::io::Result<()> { +async fn optimistic_successor_without_a_retry_ends_only_its_lease() -> std::io::Result<()> { let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await; let seq = fixture.segment_file.proxy_seq; let head_seq = seq.checked_sub(1).ok_or_else(|| std::io::Error::other("fixture head"))?; @@ -164,6 +164,7 @@ async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segmen }; let snapshot = crate::HlsLeaseManifestSnapshot { startup_revisions: Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, mode: config.mode, retained_start_seq: head_seq, revisions, @@ -195,12 +196,17 @@ async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segmen .commit_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, guard, snapshot, now) .is_committed()); } - // Listing does not expose a revision; only streaming bytes does. The origin - // failing a listed successor therefore fails just that one segment. + // Listing does not expose bytes. With no usable retry and a terminal fetch + // state, the controlled failure transition retires this lease. successor.revision().fail(); assert!(matches!( fixture.serve(None).await, - HlsResourceServeOutcome::Failure(HlsResourceServeFailure::PermanentFailed { .. }) + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing | HlsResourceServeFailure::Expired) )); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),) + .await + .is_none()); Ok(()) } diff --git a/backend/hls/src/response/tests/revision_recovery.rs b/backend/hls/src/response/tests/revision_recovery.rs new file mode 100644 index 000000000..e71f5d61b --- /dev/null +++ b/backend/hls/src/response/tests/revision_recovery.rs @@ -0,0 +1,978 @@ +use super::*; +use crate::lease::HlsRevisionRecoveryOutcome; + +async fn permanent_failure_with_counted_user(fixture: &CachedSegmentFixture) -> std::io::Result<()> { + failed_optimistic_revision(fixture, 6).await?; + assert!(fixture + .manager + .activate_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + current_time_millis(), + crate::HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = + SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None }; + let lease = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("lease"); + let mut user = tuliprox_core::model::ProxyUserCredentials::default(); + user.username = lease.username.clone(); + user.max_connections = 1; + fixture + .active_users + .create_user_session(tuliprox_session::CreateUserSessionParams { + user: &user, + session_token: &lease.user_session_token, + virtual_id: 1, + provider: "synthetic-provider", + stream_url: "http://synthetic.invalid/live.m3u8", + addr: &"127.0.0.1:12345".parse().expect("address"), + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(tuliprox_session::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + fixture.active_users.mark_grace_active(&lease.username, &lease.user_session_token).await; + let version = + fixture.active_users.grace_active_version(&lease.username, &lease.user_session_token).await.expect("version"); + fixture.active_users.activate_grace_active(&lease.username, &lease.user_session_token, version).await; + assert_eq!(fixture.active_users.user_connections(&lease.username).await, 1); + Ok(()) +} + +#[tokio::test] +async fn reentry_completion_releases_the_counted_user_reservation() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + assert_eq!( + fixture.active_users.connection_permission("test-user", 1, 0).await, + shared::model::UserConnectionPermission::Exhausted + ); + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired); + assert_eq!(fixture.active_users.user_connections("test-user").await, 0); + assert_eq!( + fixture.active_users.connection_permission("test-user", 1, 0).await, + shared::model::UserConnectionPermission::Allowed + ); + Ok(()) +} + +#[tokio::test] +async fn reentry_completion_preserves_another_counted_session_of_the_same_user() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + let mut user = tuliprox_core::model::ProxyUserCredentials::default(); + user.username = "test-user".into(); + user.max_connections = 2; + let token = "other-playback"; + fixture + .active_users + .create_user_session(tuliprox_session::CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id: 2, + provider: "synthetic-provider", + stream_url: "http://synthetic.invalid/other.m3u8", + addr: &"127.0.0.1:12346".parse().expect("address"), + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(tuliprox_session::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + fixture.active_users.mark_grace_active(&user.username, token).await; + let version = fixture.active_users.grace_active_version(&user.username, token).await.expect("version"); + fixture.active_users.activate_grace_active(&user.username, token, version).await; + assert_eq!(fixture.active_users.user_connections(&user.username).await, 2); + + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired); + assert_eq!(fixture.active_users.user_connections(&user.username).await, 1); + assert!(fixture.active_users.release_session_streams_and_counted_reservation(&user.username, token).await); + assert_eq!(fixture.active_users.user_connections(&user.username).await, 0); + Ok(()) +} + +#[tokio::test] +async fn normal_lifecycle_completion_uses_the_same_counted_user_release() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + let expires = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("lease") + .valid_until_ms + + 1; + let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default()); + fixture + .manager + .handle_lifecycle_event( + &fixture.active_users, + &ctx.active_provider, + crate::HlsLifecycleEvent { + key: crate::HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + due_at_ms: expires, + }, + expires, + ) + .await; + assert_eq!(fixture.active_users.user_connections("test-user").await, 0); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expires,) + .await + .is_none()); + Ok(()) +} + +#[tokio::test] +async fn reentry_completion_survives_request_cancellation_during_cleanup() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + fixture + .manager + .qos() + .ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None) + .await; + fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig { + max_level: shared::model::HlsSegmentRepairMode::Low, + apply_to_first_segments: 1, + ..Default::default() + }); + fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await; + let reader = fixture.session.read().await; + { + let request = recovery(&fixture); + tokio::pin!(request); + assert!(futures::poll!(request.as_mut()).is_pending()); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.manager.access_leases().try_read().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cleanup holds its lease fence while waiting for the session writer"); + assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1); + // The request deadline drops this future while cleanup is blocked. + } + drop(reader); + let settled = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if fixture.manager.qos().is_empty().await + && fixture.manager.segment_repair().stats().await.windows == 0 + && fixture.manager.pending_lease_completions_for_test() == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + assert!(settled.is_ok(), "cancelled cleanup left QoS or repair state"); + assert_eq!(fixture.active_users.user_connections("test-user").await, 0); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .is_none()); + Ok(()) +} + +#[tokio::test] +async fn concurrent_reentry_waiters_share_completion_when_one_is_cancelled() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + let guard = fixture + .manager + .access_leases() + .read() + .await + .revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq) + .expect("guard"); + let reader = fixture.session.read().await; + let second = fixture.manager.end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ); + tokio::pin!(second); + { + let first = fixture.manager.end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ); + tokio::pin!(first); + assert!(futures::poll!(first.as_mut()).is_pending()); + assert!(futures::poll!(second.as_mut()).is_pending()); + assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1); + } + drop(reader); + assert!(tokio::time::timeout(Duration::from_secs(1), second).await.expect("owned completion")); + assert_eq!(fixture.active_users.user_connections("test-user").await, 0); + assert_eq!(fixture.manager.pending_lease_completions_for_test(), 0); + assert!( + !fixture + .manager + .end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ) + .await + ); + Ok(()) +} + +#[tokio::test] +async fn stale_reentry_job_preserves_new_incarnation_and_its_user_reservation() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + let mut leases = fixture.manager.access_leases().write().await; + let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard"); + let mut replacement = + leases.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()).expect("lease"); + replacement.issued_at_ms += 1; + let replacement_issued_at = replacement.issued_at_ms; + assert!(leases.prepare_access_lease(replacement)); + drop(leases); + fixture + .manager + .qos() + .ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None) + .await; + assert!( + !fixture + .manager + .end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ) + .await + ); + assert_eq!(fixture.active_users.user_connections("test-user").await, 1); + assert_eq!(fixture.manager.qos().len().await, 1); + assert_eq!( + fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),) + .await + .expect("new incarnation") + .issued_at_ms, + replacement_issued_at + ); + Ok(()) +} + +#[tokio::test] +async fn cancelled_reentry_waiting_for_accounting_preserves_replacement_lease() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::without_lease_user_accounting(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + let guard = fixture + .manager + .access_leases() + .read() + .await + .revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq) + .expect("guard"); + { + let request = fixture.manager.end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ); + tokio::pin!(request); + assert!(futures::poll!(request.as_mut()).is_pending()); + tokio::task::yield_now().await; + assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1); + } + // The owned job outlives its caller while waiting for runtime attachment. + // It must revalidate the lease after attachment, before releasing anything. + let replacement_issued_at = { + let mut leases = fixture.manager.access_leases().write().await; + let mut replacement = leases + .response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("lease"); + replacement.issued_at_ms += 1; + let issued_at = replacement.issued_at_ms; + assert!(leases.prepare_access_lease(replacement)); + issued_at + }; + fixture.manager.bind_lease_user_accounting(&fixture.active_users); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.manager.pending_lease_completions_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("stale completion settles after runtime attachment"); + assert_eq!(fixture.active_users.user_connections("test-user").await, 1); + assert_eq!( + fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("replacement lease") + .issued_at_ms, + replacement_issued_at + ); + Ok(()) +} + +#[tokio::test] +async fn concurrent_lifecycle_and_reentry_cleanup_survive_cancellation_of_all_waiters() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + permanent_failure_with_counted_user(&fixture).await?; + fixture + .manager + .qos() + .ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None) + .await; + fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig { + max_level: shared::model::HlsSegmentRepairMode::Low, + apply_to_first_segments: 1, + ..Default::default() + }); + fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await; + let (expires_at, guard) = { + let mut leases = fixture.manager.access_leases().write().await; + let expires_at = leases + .response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("lease") + .valid_until_ms + + 1; + let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard"); + (expires_at, guard) + }; + let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default()); + let session_reader = fixture.session.read().await; + { + let lifecycle = fixture.manager.handle_lifecycle_event( + &fixture.active_users, + &ctx.active_provider, + crate::HlsLifecycleEvent { + key: crate::HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: fixture.lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + due_at_ms: expires_at, + }, + expires_at, + ); + let reentry = fixture.manager.end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis(), + ); + tokio::pin!(lifecycle, reentry); + assert!(futures::poll!(lifecycle.as_mut()).is_pending()); + assert!(futures::poll!(reentry.as_mut()).is_pending()); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.manager.access_leases().try_read().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("cleanup waits for the session writer while holding its lease fence"); + assert_eq!(fixture.manager.pending_lease_completions_for_test(), 2); + // Both caller futures are cancelled while their owned jobs are blocked. + } + drop(session_reader); + tokio::time::timeout(Duration::from_secs(1), async { + while fixture.manager.pending_lease_completions_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("owned cleanup settles after all waiters are cancelled"); + assert_eq!(fixture.active_users.user_connections("test-user").await, 0); + assert!(fixture.manager.qos().is_empty().await); + assert_eq!(fixture.manager.segment_repair().stats().await.windows, 0); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .is_none()); + Ok(()) +} + +async fn failed_optimistic_revision( + fixture: &CachedSegmentFixture, + prefix: u64, +) -> std::io::Result { + let revision = publish_revision_fixture(fixture, true).await?; + let mut session = fixture.session.write().await; + session.startup.as_mut().expect("startup").config.mode = shared::model::HlsStartupMode::Optimistic; + drop(session); + let mut leases = fixture.manager.access_leases().write().await; + let mut snapshot = leases + .response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("lease") + .last_manifest_snapshot + .expect("snapshot"); + Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).mode = + shared::model::HlsStartupMode::Optimistic; + let guard = leases + .prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("guard"); + assert!(leases + .commit_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + snapshot, + current_time_millis() + ) + .is_committed()); + revision.revision().prefix_available.store(prefix, Ordering::Release); + revision.revision().fail(); + drop(leases); + let mut session = fixture.session.write().await; + session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = SegmentCacheStatus::Fetching { + priority: crate::SegmentFetchPriority::Demand, + started_at_ms: current_time_millis(), + }; + Ok(revision) +} + +async fn install_completed_retry(fixture: &CachedSegmentFixture) -> std::io::Result { + let mut session = fixture.session.write().await; + let startup = session.startup.as_ref().expect("startup"); + let retry = startup.store.create( + fixture.proxy_session_id.clone(), + fixture.segment_file.proxy_seq, + crate::SegmentRevisionKind::Raw, + )?; + retry.revision().bind_origin(&session); + let entry = session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry"); + entry.status = SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() }; + assert!(retry.revision().complete(crate::CachedSegmentMetadata { + path: fixture.segment_cache.object_path(&entry.cache_key), + size: fixture.full_size, + })); + session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone()); + Ok(retry) +} + +async fn recovery(fixture: &CachedSegmentFixture) -> HlsRevisionRecoveryOutcome { + let marker = fixture.context.media_activity_marker.as_ref().expect("marker"); + fixture + .manager + .recover_access_lease_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + marker.lease_identity, + fixture.segment_file.proxy_seq, + current_time_millis(), + ) + .await +} + +async fn install_spooled_retry(fixture: &CachedSegmentFixture) -> std::io::Result { + use tokio::io::AsyncWriteExt; + let (store, cache_key) = { + let session = fixture.session.read().await; + ( + session.startup.as_ref().expect("startup").store.clone(), + session.segments[&fixture.segment_file.proxy_seq].cache_key.clone(), + ) + }; + let retry = store.create( + fixture.proxy_session_id.clone(), + fixture.segment_file.proxy_seq, + crate::SegmentRevisionKind::Raw, + )?; + let reservation = fixture.segment_cache.reserve_revision_peak(&cache_key, fixture.full_size * 5).await?; + let (mut file, pin) = + fixture.segment_cache.create_revision_spool_with_budget(&retry.revision().key, Some(&reservation)).await?; + let bytes = tokio::fs::read(fixture.segment_cache.object_path(&cache_key)).await?; + file.write_all(&bytes).await?; + file.flush().await?; + drop(file); + let path = fixture.segment_cache.object_path(&retry.revision().key); + fixture.segment_cache.account_reserved_spool(&path, fixture.full_size, &reservation)?; + *retry.revision().file_pin.lock().expect("pin") = Some(pin); + assert!(retry.revision().complete(crate::CachedSegmentMetadata { path, size: fixture.full_size })); + let mut session = fixture.session.write().await; + retry.revision().bind_origin(&session); + session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = + SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() }; + session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone()); + Ok(retry) +} + +#[tokio::test] +async fn partial_revision_retry_recovers_same_get_and_snapshot_without_playlist_refresh() -> std::io::Result<()> { + for prefix in [0, 6, 3_230_293] { + let fixture = CachedSegmentFixture::new(b"complete-retry-body".to_vec()).await; + let failed = failed_optimistic_revision(&fixture, prefix).await?; + let before = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("lease before"); + let availability = + fixture.manager.access_leases().read().await.availability_evidence_generation(&fixture.proxy_session_id); + let retry = install_completed_retry(&fixture).await?; + let response = ready_response(fixture.serve(None).await); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"complete-retry-body".as_slice()); + let after = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("same lease"); + let snapshot = after.last_manifest_snapshot.expect("snapshot"); + let previous_snapshot = before.last_manifest_snapshot.expect("previous snapshot"); + assert!(snapshot.snapshot_generation > previous_snapshot.snapshot_generation); + assert_eq!(snapshot.visible_segments, previous_snapshot.visible_segments); + assert_eq!(snapshot.first_proxy_seq, previous_snapshot.first_proxy_seq); + assert_eq!( + snapshot.startup_revisions.as_ref().expect("revisions").revisions[&fixture.segment_file.proxy_seq], + retry + ); + let mut leases = fixture.manager.access_leases().write().await; + assert_ne!(leases.availability_evidence_generation(&fixture.proxy_session_id), availability); + assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(retry)); + let guard = leases + .prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("guard"); + assert!(leases + .commit_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + snapshot, + current_time_millis() + ) + .is_committed()); + assert!(!failed.revision().was_exposed()); + } + Ok(()) +} + +#[tokio::test] +async fn retry_pending_is_503_without_ending_the_lease() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + failed_optimistic_revision(&fixture, 6).await?; + fixture.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25; + let failure = match fixture.serve(None).await { + HlsResourceServeOutcome::Failure(failure) => failure, + HlsResourceServeOutcome::Ready(_) => panic!("retry must wait"), + }; + let response = crate::response::cache::hls_resource_failure_default_response(failure); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(header::RETRY_AFTER)); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .is_some()); + install_completed_retry(&fixture).await?; + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound); + Ok(()) +} + +#[tokio::test] +async fn waiting_get_resolves_a_retry_that_finishes_before_its_deadline() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + failed_optimistic_revision(&fixture, 6).await?; + let request = fixture.serve(None); + let complete = async { + tokio::time::sleep(Duration::from_millis(30)).await; + install_completed_retry(&fixture).await + }; + let (response, retry) = tokio::join!(request, complete); + retry?; + let response = ready_response(response); + assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"retry-body".as_slice()); + Ok(()) +} + +#[tokio::test] +async fn exposed_failure_ends_only_the_old_lease_and_cannot_append_retry_bytes() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + let old = publish_revision_fixture(&fixture, true).await?; + let base_snapshot = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("old lease") + .last_manifest_snapshot + .expect("old snapshot"); + let response = ready_response(fixture.serve(None).await); + let mut stream = response.into_body().into_data_stream(); + assert_eq!(stream.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice())); + old.revision().fail(); + let retry = install_completed_retry(&fixture).await?; + assert!(stream.next().await.is_some_and(|chunk| chunk.is_err())); + assert!(stream.next().await.is_none()); + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired); + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .is_none()); + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::Unchanged); + assert!(fixture.manager.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.is_some()); + assert_eq!( + fixture.session.read().await.startup.as_ref().expect("shared startup").revisions + [&fixture.segment_file.proxy_seq], + retry + ); + let new_id = HlsAccessLeaseId("replacement-viewer".into()); + let marker = live_media_marker(&fixture.manager, &fixture.session, new_id.clone()).await; + let mut leases = fixture.manager.access_leases().write().await; + assert!(leases.media_identity_is_current( + &new_id, + &fixture.proxy_session_id, + marker.lease_identity, + current_time_millis() + )); + let mut snapshot = base_snapshot; + snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + window_plan: None, + mode: shared::model::HlsStartupMode::Optimistic, + retained_start_seq: fixture.segment_file.proxy_seq, + revisions: [(fixture.segment_file.proxy_seq, retry)].into_iter().collect(), + })); + let guard = leases + .prepare_manifest_publication(&new_id, &fixture.proxy_session_id, current_time_millis()) + .expect("new publication"); + assert!(leases + .commit_manifest_publication(&new_id, &fixture.proxy_session_id, guard, snapshot, current_time_millis()) + .is_committed()); + drop(leases); + let context = HlsCacheResponseContext::new( + new_id, + test_log_identity(), + 300, + fixture.manager.metrics().clone(), + fixture.manager.segment_repair().clone(), + None, + Some(marker), + current_time_millis(), + ); + let response = ready_response( + serve_hls_segment_cache_outcome( + fixture.segment_cache.clone(), + fixture.session.clone(), + fixture.segment_file.clone(), + None, + &context, + ) + .await, + ); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("new lease body").to_bytes(), b"retry-body".as_slice()); + Ok(()) +} + +#[tokio::test] +async fn recovery_fences_generation_snapshot_and_lease_incarnation() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + failed_optimistic_revision(&fixture, 6).await?; + let guard = fixture + .manager + .access_leases() + .read() + .await + .revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq) + .expect("guard"); + install_completed_retry(&fixture).await?; + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound); + let mut leases = fixture.manager.access_leases().write().await; + let session = fixture.session.read().await; + assert_eq!( + leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()), + HlsRevisionRecoveryOutcome::Unchanged + ); + assert!(leases + .end_revision_playback_if_current(&fixture.lease_id, &guard, &session, current_time_millis()) + .is_none()); + drop(session); + drop(leases); + // Replacing the same ID does not let a delayed failure end the new viewer. + live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await; + assert!( + !fixture + .manager + .end_access_lease_for_failed_revision( + &fixture.lease_id, + &fixture.proxy_session_id, + &guard, + current_time_millis() + ) + .await + ); + Ok(()) +} + +#[tokio::test] +async fn concurrent_recovery_commits_once_and_preserves_progressive_quota() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + failed_optimistic_revision(&fixture, 6).await?; + { + let mut leases = fixture.manager.access_leases().write().await; + assert!(leases.try_claim_progressive_startup( + &fixture.lease_id, + fixture.segment_file.proxy_seq, + current_time_millis() + )); + } + install_completed_retry(&fixture).await?; + let (first, second) = tokio::join!(recovery(&fixture), recovery(&fixture)); + assert!(matches!( + (first, second), + (HlsRevisionRecoveryOutcome::ReplacementBound, HlsRevisionRecoveryOutcome::Unchanged) + | (HlsRevisionRecoveryOutcome::Unchanged, HlsRevisionRecoveryOutcome::ReplacementBound) + )); + assert!(!fixture.manager.access_leases().write().await.try_claim_progressive_startup( + &fixture.lease_id, + fixture.segment_file.proxy_seq, + current_time_millis() + )); + Ok(()) +} + +#[tokio::test] +async fn origin_generation_and_mapping_changes_never_install_retired_retry_bytes() -> std::io::Result<()> { + for change_generation in [false, true] { + let fixture = CachedSegmentFixture::new(b"retired-retry".to_vec()).await; + let failed = failed_optimistic_revision(&fixture, 6).await?; + install_completed_retry(&fixture).await?; + let mut leases = fixture.manager.access_leases().write().await; + let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard"); + let mut session = fixture.session.write().await; + if change_generation { + session.activity.origin_work_generation += 1; + } else { + session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").origin_key.origin_epoch += 1; + } + assert_eq!( + leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()), + HlsRevisionRecoveryOutcome::ReentryRequired + ); + assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed)); + } + Ok(()) +} + +#[tokio::test] +async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + let failed = failed_optimistic_revision(&fixture, 6).await?; + install_completed_retry(&fixture).await?; + let mut leases = fixture.manager.access_leases().write().await; + let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard"); + let expires = leases + .response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .expect("lease") + .valid_until_ms; + let session = fixture.session.read().await; + assert_eq!( + leases.recover_revision_binding(&fixture.lease_id, &guard, &session, expires + 1), + HlsRevisionRecoveryOutcome::Unchanged + ); + assert!(leases.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, expires + 1).is_none()); + assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed)); + Ok(()) +} + +#[tokio::test] +async fn fetch_completion_recovers_before_get_and_final_failure_reuses_manager_cleanup() -> std::io::Result<()> { + for succeeds in [false, true] { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + let failed = failed_optimistic_revision(&fixture, 6).await?; + let generation = fixture.session.read().await.activity.origin_work_generation; + fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig { + max_level: shared::model::HlsSegmentRepairMode::Low, + apply_to_first_segments: 1, + ..Default::default() + }); + fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await; + fixture + .manager + .qos() + .ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None) + .await; + let other_id = HlsAccessLeaseId("unaffected-viewer".into()); + let other = live_media_marker(&fixture.manager, &fixture.session, other_id.clone()).await; + fixture.manager.segment_repair().start_access_lease_window(other_id.clone()).await; + fixture + .manager + .qos() + .ensure_access_lease(&other_id, &fixture.proxy_session_id, current_time_millis(), None) + .await; + let retry = if succeeds { + Some(install_completed_retry(&fixture).await?) + } else { + fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = + SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None }; + None + }; + let runtime = fixture.manager.segment_worker_pool().revision_recovery.get().expect("manager recovery runtime"); + runtime + .recover_segment_leases_after_fetch( + &fixture.proxy_session_id, + fixture.segment_file.proxy_seq, + generation.wrapping_sub(1), + current_time_millis(), + ) + .await; + assert_eq!( + fixture + .manager + .access_leases() + .read() + .await + .published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), + Some(failed) + ); + runtime + .recover_segment_leases_after_fetch( + &fixture.proxy_session_id, + fixture.segment_file.proxy_seq, + generation, + current_time_millis(), + ) + .await; + if succeeds { + assert_eq!( + fixture + .manager + .access_leases() + .read() + .await + .published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), + retry + ); + assert_eq!(fixture.manager.segment_repair().stats().await.windows, 2); + assert_eq!(fixture.manager.qos().len().await, 2); + } else { + assert!(fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .is_none()); + assert_eq!(fixture.manager.segment_repair().stats().await.windows, 1); + assert_eq!(fixture.manager.qos().len().await, 1); + } + assert!(fixture.manager.access_leases().write().await.media_identity_is_current( + &other_id, + &fixture.proxy_session_id, + other.lease_identity, + current_time_millis(), + )); + } + Ok(()) +} + +#[tokio::test] +async fn playlist_publication_uses_the_same_recovery_and_never_reinstalls_an_older_attempt() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + let failed = failed_optimistic_revision(&fixture, 6).await?; + let old_snapshot = fixture + .manager + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("lease") + .last_manifest_snapshot + .expect("snapshot"); + let retry = install_completed_retry(&fixture).await?; + let guard = fixture + .manager + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("publication"); + assert!(fixture + .manager + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + old_snapshot.clone(), + current_time_millis() + ) + .await + .is_committed()); + assert_eq!( + fixture + .manager + .access_leases() + .read() + .await + .published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), + Some(retry) + ); + // An old request can still contain the failed guard, but it cannot rewind + // the successful transaction to that guard or make it complete again. + assert!(!failed.revision().complete(crate::CachedSegmentMetadata { path: "retired.ts".into(), size: 1 })); + let guard = fixture + .manager + .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) + .await + .expect("publication"); + assert!(fixture + .manager + .commit_access_lease_manifest_publication( + &fixture.lease_id, + &fixture.proxy_session_id, + guard, + old_snapshot, + current_time_millis() + ) + .await + .is_committed()); + Ok(()) +} + +#[tokio::test] +async fn repeated_recovery_cycles_release_old_publication_pins_and_replay_budget() -> std::io::Result<()> { + let mut fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; + for _ in 0..128 { + let marker = live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await; + fixture.context.media_activity_marker = Some(marker); + let failed = failed_optimistic_revision(&fixture, 6).await?; + let (store, budget) = { + let session = fixture.session.read().await; + let startup = session.startup.as_ref().expect("startup"); + (startup.store.clone(), startup.budget.clone()) + }; + let retry = install_spooled_retry(&fixture).await?; + assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound); + assert_eq!(failed.revision().pin_count(), 1, "no failed guard remains in the lease or snapshot"); + assert_eq!(budget.usage(), (0, 0)); + fixture.manager.access_leases().write().await.remove_access_lease(&fixture.lease_id); + fixture.session.write().await.startup.as_mut().expect("startup").revisions.clear(); + drop(failed); + drop(retry); + let retired = store.retire_unpinned()?; + assert_eq!(retired.len(), 2); + for revision in retired { + fixture.segment_cache.delete_if_inactive(&revision.key).await?; + assert!(!fixture.segment_cache.object_path(&revision.key).exists()); + } + assert!(store.retire_unpinned()?.is_empty()); + assert_eq!(fixture.segment_cache.revision_disk_bytes(), 0); + assert_eq!(fixture.segment_cache.remaining_revision_bytes(&fixture.proxy_session_id), 0); + assert!(!fixture + .segment_cache + .has_active_temp_files_for_session_except::(&fixture.proxy_session_id, &[])); + } + Ok(()) +} diff --git a/backend/hls/src/segment_fetcher.rs b/backend/hls/src/segment_fetcher.rs index 26df1a71f..279b365b0 100644 --- a/backend/hls/src/segment_fetcher.rs +++ b/backend/hls/src/segment_fetcher.rs @@ -87,6 +87,7 @@ pub enum HlsSwitchResourceKind { /// Bounded scheduler for live HLS segment demand fetches and prefetches. pub struct HlsSegmentWorkerPool { runtime: ArcSwap, + pub(crate) revision_recovery: std::sync::OnceLock>, access_leases: Arc>, metrics: Arc, availability_reevaluations: Option>, diff --git a/backend/hls/src/segment_fetcher/commit.rs b/backend/hls/src/segment_fetcher/commit.rs index 9564cfe87..060a9b38f 100644 --- a/backend/hls/src/segment_fetcher/commit.rs +++ b/backend/hls/src/segment_fetcher/commit.rs @@ -12,6 +12,7 @@ pub(super) struct SegmentFetchCompletion { pub(super) notifier: Option>, pub(super) scheduled_retry: Option, pub(super) evidence_changed_for: Option, + pub(super) recovery_generation: Option, } pub(super) struct SegmentFetchCommit { @@ -113,6 +114,7 @@ impl HlsSegmentWorkerPool { notifier, scheduled_retry, evidence_changed_for, + recovery_generation: generation_valid.then_some(session.activity.origin_work_generation), } } @@ -157,7 +159,8 @@ pub(super) fn commit_failed_segment_fetch( error: &SegmentFetchError, finished_at_ms: u64, ) { - let missing_candidate = session.origin_source.archive_reference.is_none() + let missing_candidate = !session.uses_fast_start_live_window() + && session.origin_source.archive_reference.is_none() && session .publishable_origin_head_proxy_seq .is_some_and(|head| snapshot.proxy_seq >= head && snapshot.proxy_seq <= missing_head_bound(session, head)) @@ -206,6 +209,9 @@ fn missing_head_bound(session: &super::super::HlsSession, head: u64) -> u64 { /// new origin window: until a client has seen that head, skipping a missing /// object cannot open a hole in published media. pub(crate) fn recompute_unpublished_live_head(session: &mut super::super::HlsSession) { + if session.uses_fast_start_live_window() { + return; + } if session.origin_source.archive_reference.is_some() { return; } diff --git a/backend/hls/src/segment_fetcher/scheduler.rs b/backend/hls/src/segment_fetcher/scheduler.rs index fbc88bd33..aeaac2831 100644 --- a/backend/hls/src/segment_fetcher/scheduler.rs +++ b/backend/hls/src/segment_fetcher/scheduler.rs @@ -186,6 +186,7 @@ impl HlsSegmentWorkerPool { ) -> Self { Self { runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)), + revision_recovery: std::sync::OnceLock::new(), access_leases, metrics, availability_reevaluations, @@ -326,11 +327,19 @@ impl HlsSegmentWorkerPool { if mode.publishes_pending_successors() { return false; } - let Some(head) = session.publishable_origin_head_proxy_seq else { - return true; + let heads = if session.uses_fast_start_live_window() { + session.live_window_heads() + } else { + session.publishable_origin_head_proxy_seq.into_iter().collect() }; - startup.revisions.get(&head).is_none_or(|revision| { - mode != shared::model::HlsStartupMode::Conservative && !revision.revision().is_startup_head_ready(mode) + if heads.is_empty() { + return true; + } + heads.iter().any(|head| { + startup.revisions.get(head).is_none_or(|revision| { + mode != shared::model::HlsStartupMode::Conservative + && !revision.revision().is_startup_head_ready(mode) + }) }) }); let fetch_limit = if initial_barrier { 1 } else { policy.max_session_segment_fetches }; @@ -404,6 +413,20 @@ impl HlsSegmentWorkerPool { let mut session = context.session.write().await; self.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, finished_at_ms) }; + if let Some(recovery_generation) = completion.recovery_generation { + if let Some(recovery) = self.revision_recovery.get() { + let session_id = context.session.read().await.proxy_session_id.clone(); + recovery + .recover_segment_leases_after_fetch( + &session_id, + snapshot.proxy_seq, + recovery_generation, + finished_at_ms, + ) + .await; + } + } + let mut notifier = completion.notifier; if completion.stale_commit_cleanup_reserved { if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await { diff --git a/backend/hls/src/segment_fetcher/startup_fill.rs b/backend/hls/src/segment_fetcher/startup_fill.rs index f83c8a0d9..6784c741c 100644 --- a/backend/hls/src/segment_fetcher/startup_fill.rs +++ b/backend/hls/src/segment_fetcher/startup_fill.rs @@ -447,12 +447,13 @@ async fn bind_startup_revision( ))); } let decides_policy = startup_fill_decides_policy(&session, snapshot.proxy_seq); + revision.revision().bind_origin(&session); let Some(startup) = &mut session.startup else { return Ok(()) }; startup.policy_fixed |= decides_policy; startup.revisions.insert(snapshot.proxy_seq, revision.clone()); // An optimistic window may become complete when its last successor is // accepted, after the head already has bytes. - if startup.config.mode.publishes_pending_successors() && session.last_rendered_manifest.is_none() { + if startup.config.mode.publishes_pending_successors() { if let Err(error) = session.render_and_store_manifest(current_time_millis()) { debug!( "HLS optimistic startup render after successor binding failed: proxy_seq={} error={error:?}", @@ -467,6 +468,9 @@ async fn bind_startup_revision( /// fill may fix the session policy or fall back to `first_ready`. A successor /// without replay stays raw and is delivered once complete. fn startup_fill_decides_policy(session: &super::super::HlsSession, proxy_seq: u64) -> bool { + if session.uses_fast_start_live_window() { + return session.live_window_heads().contains(&proxy_seq); + } !session.startup_mode().publishes_pending_successors() || session.publishable_origin_head_proxy_seq == Some(proxy_seq) } diff --git a/backend/hls/src/segment_fetcher/tests.rs b/backend/hls/src/segment_fetcher/tests.rs index dea31ae05..38719ad7c 100644 --- a/backend/hls/src/segment_fetcher/tests.rs +++ b/backend/hls/src/segment_fetcher/tests.rs @@ -7,6 +7,7 @@ use super::{ mod admission; mod behavior; mod lifecycle; +mod live_window; mod persistence; mod protocol; mod query; diff --git a/backend/hls/src/segment_fetcher/tests/live_window.rs b/backend/hls/src/segment_fetcher/tests/live_window.rs new file mode 100644 index 000000000..6fbbb1410 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/live_window.rs @@ -0,0 +1,423 @@ +use super::{ + fetch_context, install_startup, normal_manifest, spawn_segment_server, SegmentFetchPolicy, SegmentFetchPriority, +}; +use crate::{ + api::{HlsLiveWindowView, HlsManifestRenderer}, + CachedSegmentMetadata, HlsSession, HlsSessionKey, SegmentCacheStatus, SegmentRevisionKind, +}; +use shared::model::{HlsStartupMode, HlsStripMode}; +use std::{fmt::Write as _, sync::atomic::Ordering}; +use tuliprox_core::model::StripConfig; + +const MODES: [HlsStartupMode; 3] = + [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic]; + +fn manifest( + origin: &str, + first: u64, + durations: &[u64], +) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-MEDIA-SEQUENCE:{first}\n"); + for (offset, duration) in durations.iter().enumerate() { + writeln!(body, "#EXTINF:{duration},\n{origin}/{}.ts", first + u64::try_from(offset).unwrap()).unwrap(); + } + normal_manifest(&body) +} + +fn prepare_selected(session: &mut HlsSession, sequences: &[u64], mode: HlsStartupMode, complete: bool) { + let startup = session.startup.as_mut().unwrap(); + for (index, seq) in sequences.iter().enumerate() { + let revision = startup + .store + .create( + session.proxy_session_id.clone(), + *seq, + if mode == HlsStartupMode::FirstReady { + SegmentRevisionKind::Processed + } else { + SegmentRevisionKind::Raw + }, + ) + .unwrap(); + if complete || (mode == HlsStartupMode::FirstReady && index == 0) { + revision.revision().complete(CachedSegmentMetadata { path: std::path::PathBuf::from("unused"), size: 1 }); + } else if index == 0 || mode != HlsStartupMode::Optimistic { + revision.revision().prefix_available.store(1, Ordering::Release); + } + startup.revisions.insert(*seq, revision); + } +} + +#[tokio::test] +async fn live_window_matrix_renders_only_selected_revisions_and_selects_real_fetch_snapshots() { + let server = spawn_segment_server(0).await; + for mode in MODES { + for (count, strip_mode, strip, offset, expected) in [ + (6, HlsStripMode::Segments, 0, 0, vec![3, 4, 5]), + (6, HlsStripMode::Segments, 1, 0, vec![2, 3, 4]), + (6, HlsStripMode::Segments, 2, 0, vec![1, 2, 3]), + (6, HlsStripMode::Segments, 3, 0, vec![0, 1, 2]), + (6, HlsStripMode::Segments, u64::MAX, 0, vec![0, 1, 2]), + (12, HlsStripMode::Segments, 1, 0, vec![8, 9, 10]), + (12, HlsStripMode::Segments, 1, 9_000_000, vec![9_000_008, 9_000_009, 9_000_010]), + (6, HlsStripMode::Seconds, 0, 0, vec![3, 4, 5]), + (6, HlsStripMode::Seconds, 7, 0, vec![2, 3, 4]), + (6, HlsStripMode::Seconds, 8, 0, vec![1, 2, 3]), + (6, HlsStripMode::Seconds, 10, 0, vec![1, 2, 3]), + (6, HlsStripMode::Seconds, u64::MAX, 0, vec![0, 1, 2]), + ] { + for parallelism in [1, 2, 3] { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy { + max_session_segment_fetches: parallelism, + max_prefetch_queue_depth: 0, + ..Default::default() + }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + state.proxy_next_seq = Some(offset); + let durations = if count == 6 { vec![4, 4, 4, 4, 3, 7] } else { vec![4; count] }; + state.apply_origin_manifest(&manifest(&server.base_url, 100_000, &durations)).unwrap(); + state.live_window_strip = StripConfig { mode: strip_mode, value: strip }; + state.configure_segment_prefetch_queue(0); + } + install_startup(&context, &worker, mode).await; + let plan = { + let mut state = context.session.write().await; + let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap(); + assert_eq!(plan.visible, expected, "mode={mode} strip={strip} offset={offset}"); + state.queue_manifest_fetch_candidates(10, false); + assert_eq!(crate::renderer_candidate_window_proxy_seqs(&state), expected); + let protected = crate::gc::ProtectedSet::from_session(&state); + assert_eq!( + protected.segment_proxy_seqs.into_iter().collect::>(), + expected.iter().copied().collect() + ); + plan + }; + // Exercise the real scheduler, including the entry barrier. Its + // next fetch must be the chosen head, never an older origin URI. + for expected_seq in &expected { + let snapshot = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap(); + assert_eq!(snapshot.proxy_seq, *expected_seq); + assert_eq!(snapshot.origin_seq, 100_000 + expected_seq - offset); + assert_eq!(snapshot.priority, SegmentFetchPriority::RenderWindow); + let mut state = context.session.write().await; + state.active_segment_fetches -= 1; + prepare_selected(&mut state, &[*expected_seq], mode, true); + } + assert!(worker.next_fetch_snapshot(&context, 12, &policy).await.is_none()); + let mut state = context.session.write().await; + prepare_selected(&mut state, &expected, mode, false); + let rendered = HlsManifestRenderer::render_live_window(&state, &plan, 12).unwrap(); + assert_eq!(rendered.segment_proxy_seqs, expected); + assert_eq!(state.startup.as_ref().unwrap().revisions.len(), 3); + assert!(rendered.body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", expected[0]))); + } + } + } +} + +#[tokio::test] +async fn live_window_short_empty_readiness_and_stale_plans_are_explicit() { + let server = spawn_segment_server(0).await; + for mode in MODES { + for count in 0..=3 { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + state.apply_origin_manifest(&manifest(&server.base_url, 0, &vec![4; count])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Seconds, value: u64::MAX }; + } + install_startup(&context, &worker, mode).await; + let mut state = context.session.write().await; + state.record_live_window_origin_commit(0); + let Some(plan) = state.plan_live_window(HlsLiveWindowView::Entry) else { + assert_eq!(count, 0); + continue; + }; + assert_eq!(plan.visible.len(), count); + assert_eq!(plan.effective_strip_segments, 0); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err()); + prepare_selected(&mut state, &plan.visible, mode, false); + // Qualification of a one/two-segment source is still mandatory. + assert_eq!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok(), count == 3); + state.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Qualified { + basis: crate::short_manifest::HlsOriginQualificationBasis::capture(&state), + generation: crate::manifest_acceptance::HlsManifestAcceptanceGeneration(1), + windows: crate::short_manifest::HlsQualifiedOriginWindows::ShortAllowed, + publication_boundary: None, + }; + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok()); + state.activity.origin_work_generation += 1; + assert!(!plan.is_current(&state)); + assert!(state.plan_live_window(HlsLiveWindowView::Entry).is_none()); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err()); + } + } +} + +#[tokio::test] +async fn live_window_http_fills_selected_head_with_existing_progressive_delivery() { + assert_live_window_http_fill(None).await; +} + +#[tokio::test] +#[ignore = "manual probe: set TULIPROX_HLS_WINDOW_ORIGIN to a controlled streaming fixture"] +async fn live_window_http_fills_selected_head_against_manual_origin() { + let origin = std::env::var("TULIPROX_HLS_WINDOW_ORIGIN").expect("manual origin URL must be configured"); + assert_live_window_http_fill(Some(&origin)).await; +} + +async fn assert_live_window_http_fill(external: Option<&str>) { + let server = spawn_segment_server(0).await; + let base = external.unwrap_or(&server.base_url); + for mode in MODES { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy { max_prefetch_queue_depth: 0, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let origin = if external.is_some() { + let url = format!("{base}/index.m3u8"); + let body = + context.client.get(&url).send().await.unwrap().error_for_status().unwrap().text().await.unwrap(); + match tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(&body, &url) { + tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) => manifest, + tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::TransientPassthrough { + .. + } => panic!("expected ordinary live origin"), + } + } else { + manifest(base, 0, &[4; 6]) + }; + state.apply_origin_manifest(&origin).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + state.configure_segment_prefetch_queue(0); + } + install_startup(&context, &worker, mode).await; + context.session.write().await.startup.as_mut().unwrap().worker = std::sync::Weak::new(); + super::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + tuliprox_core::utils::current_time_millis(), + ) + .await; + context.session.write().await.queue_manifest_fetch_candidates(10, false); + for seq in [2, 3, 4] { + let snapshot = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap(); + assert_eq!(snapshot.proxy_seq, seq); + let task_context = context.clone(); + let task_snapshot = snapshot.clone(); + let task_policy = policy.clone(); + let task = tokio::spawn(async move { + super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await + }); + if external.is_some() && mode != HlsStartupMode::FirstReady { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if context + .session + .read() + .await + .startup + .as_ref() + .unwrap() + .revisions + .get(&seq) + .is_some_and(|revision| revision.revision().is_publishable()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&seq].clone(); + let mut response = + crate::progressive_startup::revision_body(revision, std::time::Duration::from_secs(5), 0, None); + let prefix = futures::StreamExt::next(&mut response).await.unwrap().unwrap(); + assert!(prefix.starts_with(b"segment-")); + assert!(prefix.len() < 12); + assert!(!task.is_finished(), "raw prefix must arrive before held origin EOF"); + } + let metadata = task.await.unwrap().unwrap(); + assert!(metadata.content_length > 0); + let mut state = context.session.write().await; + state.active_segment_fetches -= 1; + state.segments.get_mut(&seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: metadata.content_length, ready_at_ms: 12 }; + } + let state = context.session.read().await; + let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap(); + assert_eq!( + HlsManifestRenderer::render_live_window(&state, &plan, 12).unwrap().segment_proxy_seqs, + vec![2, 3, 4] + ); + } +} + +#[tokio::test] +async fn live_window_readiness_uses_selected_head_and_never_skips_a_failure() { + let server = spawn_segment_server(0).await; + for mode in MODES { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, mode).await; + let mut state = context.session.write().await; + state.apply_origin_manifest(&manifest(&server.base_url, 1, &[4; 6])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap(); + let head = plan.visible[0]; + prepare_selected(&mut state, &plan.visible, mode, false); + let startup = state.startup.as_mut().unwrap(); + let entry = startup.revisions.remove(&head).unwrap(); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err()); + state.startup.as_mut().unwrap().revisions.insert(head, entry); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok()); + let successor = plan.visible[1]; + state.startup.as_ref().unwrap().revisions[&successor].revision().fail(); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err()); + state.segments.get_mut(&head).unwrap().status = + SegmentCacheStatus::FailedPermanent { failed_at_ms: 0, status: Some(axum::http::StatusCode::NOT_FOUND) }; + let origin_head = state.publishable_origin_head_proxy_seq; + super::super::recompute_unpublished_live_head(&mut state); + assert_eq!(state.publishable_origin_head_proxy_seq, origin_head); + assert_eq!(state.plan_live_window(HlsLiveWindowView::Entry).unwrap().visible, plan.visible); + state.apply_origin_manifest(&manifest(&server.base_url, 2, &[4; 6])).unwrap(); + assert!(!plan.is_current(&state)); + assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err()); + state.apply_origin_manifest(&manifest(&server.base_url, 8, &[])).unwrap(); + assert!(state.uses_fast_start_live_window()); + assert!(state.plan_live_window(HlsLiveWindowView::Entry).is_none()); + assert!(HlsManifestRenderer::render(&state, 0).is_err()); + } +} + +#[tokio::test] +async fn live_window_active_successors_are_required_with_zero_reserve_and_demand_keeps_priority() { + let server = spawn_segment_server(0).await; + for mode in MODES { + let directory = tempfile::tempdir().unwrap(); + let policy = + SegmentFetchPolicy { max_prefetch_queue_depth: 0, max_session_segment_fetches: 3, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + state.configure_segment_prefetch_queue(0); + } + install_startup(&context, &worker, mode).await; + { + let mut state = context.session.write().await; + prepare_selected(&mut state, &[2, 3, 4], mode, true); + for seq in [2, 3, 4] { + state.segments.get_mut(&seq).unwrap().status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 }; + } + state.live_window_demands.insert( + crate::HlsAccessLeaseId("active".into()), + crate::live_window::HlsLiveWindowDemand { + issued_at_ms: 0, + view: HlsLiveWindowView::Active { first_published: 2 }, + published: vec![2, 3, 4], + }, + ); + let report = state.queue_manifest_fetch_candidates(10, false); + assert_eq!(report.render_window_queued, 1); + assert_eq!(report.prefetch_queued, 0); + assert!(state.queue_segment_fetch_candidate(0, SegmentFetchPriority::Demand, 10)); + } + let demand = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap(); + assert_eq!((demand.proxy_seq, demand.priority), (0, SegmentFetchPriority::Demand)); + let successor = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap(); + assert_eq!((successor.proxy_seq, successor.priority), (5, SegmentFetchPriority::RenderWindow)); + assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none()); + } +} + +#[tokio::test] +async fn live_window_entry_barrier_and_parallel_limits_follow_the_selected_head() { + let server = spawn_segment_server(0).await; + for mode in MODES { + for parallelism in [1, 2, 3] { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy { max_session_segment_fetches: parallelism, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + } + install_startup(&context, &worker, mode).await; + context.session.write().await.queue_manifest_fetch_candidates(10, false); + assert_eq!(worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap().proxy_seq, 2); + if mode != HlsStartupMode::Optimistic { + assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none()); + prepare_selected(&mut *context.session.write().await, &[2], mode, false); + } + for seq in 3..2 + u64::try_from(parallelism).unwrap() { + assert_eq!(worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap().proxy_seq, seq); + } + assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none()); + assert_eq!(context.session.read().await.active_segment_fetches, parallelism); + } + } +} + +#[tokio::test] +async fn live_window_raw_fallback_is_decided_by_the_selected_head_even_with_an_older_demand() { + let server = spawn_segment_server(0).await; + for mode in [HlsStartupMode::Progressive, HlsStartupMode::Optimistic] { + let directory = tempfile::tempdir().unwrap(); + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut state = context.session.write().await; + *state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap(); + state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 }; + } + install_startup(&context, &worker, mode).await; + { + let mut state = context.session.write().await; + let startup = state.startup.as_mut().unwrap(); + startup.config.max_progressive_bytes_per_segment = shared::model::Bytes::new(4); + startup.worker = std::sync::Weak::new(); + state.queue_manifest_fetch_candidates(10, false); + state.queue_segment_fetch_candidate(0, SegmentFetchPriority::Demand, 10); + } + super::grant_usable_worker_access_lease_at( + &worker, + &context.session.read().await.proxy_session_id, + tuliprox_core::utils::current_time_millis(), + ) + .await; + for seq in [0, 2] { + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + assert_eq!(snapshot.proxy_seq, seq); + let result = super::super::fetch_segment_into_cache(&context, &snapshot, &policy).await; + assert!(result.is_ok()); + let mut state = context.session.write().await; + worker.apply_segment_fetch_result(&mut state, &snapshot, &policy, result, 11); + if seq == 0 { + assert_eq!(state.startup_mode(), mode, "an older demand cannot choose the entry fallback"); + assert!(!state.startup.as_ref().unwrap().policy_fixed); + } else { + assert_eq!(state.startup_mode(), HlsStartupMode::FirstReady); + assert!(state.startup.as_ref().unwrap().policy_fixed); + } + } + } +} diff --git a/backend/hls/src/segment_fetcher/tests/startup.rs b/backend/hls/src/segment_fetcher/tests/startup.rs index 0a2afd23b..4dbd53e10 100644 --- a/backend/hls/src/segment_fetcher/tests/startup.rs +++ b/backend/hls/src/segment_fetcher/tests/startup.rs @@ -250,3 +250,114 @@ async fn optimistic_successor_replay_rejection_keeps_the_session_policy() -> std assert!(prepared_head.raw); Ok(()) } + +#[tokio::test] +async fn optimistic_binding_renders_cold_and_retained_sessions_in_both_event_orders() -> std::io::Result<()> { + for (retained, head_first) in [(false, false), (false, true), (true, false), (true, true)] { + // All origin input is supplied in memory; this probe opens no HTTP connection. + let server = super::TestSegmentServer { + base_url: "http://127.0.0.1:9".into(), + requests: Arc::new(tokio::sync::Mutex::new(Vec::new())), + task: tokio::spawn(async {}), + }; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy { max_session_segment_fetches: 3, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + { + let mut session = context.session.write().await; + if retained { + for entry in session.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }; + } + session.render_and_store_manifest(1).unwrap(); + assert_eq!(session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, 1); + } + session.apply_origin_manifest(&super::normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4\n#EXTINF:4,\n4.ts\n#EXTINF:4,\n5.ts\n#EXTINF:4,\n6.ts\n" + )).unwrap(); + while session.segment_prefetch_queue.pop_next().is_some() {} + for entry in session.segments.values_mut() { + if matches!(entry.status, SegmentCacheStatus::Queued { .. }) { + entry.status = SegmentCacheStatus::Discovered; + } + } + for seq in [4, 5, 6] { + assert!(session.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Demand, 10)); + } + } + install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await; + context.session.write().await.startup.as_mut().unwrap().worker = std::sync::Weak::new(); + let mut snapshots = std::collections::BTreeMap::new(); + for _ in 0..3 { + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); + snapshots.insert(snapshot.proxy_seq, snapshot); + } + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let head = super::super::prepare_startup_fill(&context, &snapshots[&4], Some(2), deadline) + .await + .map_err(|e| std::io::Error::other(format!("{e:?}")))?; + if head_first { + head.observed.lock().await.observe(b"a").await?; + } + let mut successors = Vec::new(); + for seq in [5, 6] { + let successor = super::super::prepare_startup_fill(&context, &snapshots[&seq], Some(2), deadline) + .await + .map_err(|e| std::io::Error::other(format!("{e:?}")))?; + successors.push(successor); + } + if !head_first { + let session = context.session.read().await; + assert_eq!( + session.last_rendered_manifest.as_ref().map(|manifest| manifest.first_proxy_seq), + retained.then_some(1) + ); + drop(session); + head.observed.lock().await.observe(b"a").await?; + } + { + let session = context.session.read().await; + assert_eq!(crate::HlsManifestRenderer::render(&session, 20).unwrap().first_proxy_seq, 4); + let stored = session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq; + assert_eq!(stored, 4, "retained={retained} head_first={head_first}"); + assert!(session + .startup + .as_ref() + .unwrap() + .revisions + .values() + .all(|owner| matches!(owner.revision().state(), crate::SegmentRevisionState::Pending))); + } + // A second body chunk is not another render event. + let rendered_at = context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms; + head.observed.lock().await.observe(b"b").await?; + assert_eq!(context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms, rendered_at); + assert!(server.requests.lock().await.is_empty()); + } + Ok(()) +} + +#[tokio::test] +async fn retired_generation_cannot_bind_or_render_an_optimistic_revision() -> std::io::Result<()> { + let server = spawn_sequence_response_server(Vec::new()).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await; + let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.expect("snapshot"); + context.session.write().await.activity.origin_work_generation += 1; + let result = super::super::prepare_startup_fill( + &context, + &snapshot, + Some(2), + tokio::time::Instant::now() + Duration::from_secs(1), + ) + .await; + assert!(result.is_err()); + let session = context.session.read().await; + assert!(session.startup.as_ref().expect("startup").revisions.is_empty()); + assert!(session.last_rendered_manifest.is_none()); + assert_eq!(context.segment_cache.remaining_revision_bytes(&session.proxy_session_id), 0); + assert_eq!(super::temp_cache_files(directory.path()).len(), 0); + Ok(()) +} diff --git a/backend/hls/src/segment_fetcher/tests/startup_capacity.rs b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs index 9ca91015b..2a2b14766 100644 --- a/backend/hls/src/segment_fetcher/tests/startup_capacity.rs +++ b/backend/hls/src/segment_fetcher/tests/startup_capacity.rs @@ -11,20 +11,31 @@ async fn startup_failure_and_cancellation_close_each_commit_transition() -> std: StartupWritePhase::AfterPublication, ] { for cancel in [false, true] { - startup_transition_round(phase, cancel).await?; + for head in [1, 3] { + startup_transition_round(phase, cancel, head).await?; + } } } Ok(()) } -async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std::io::Result<()> { +async fn startup_transition_round(phase: StartupWritePhase, cancel: bool, head: u64) -> std::io::Result<()> { let server = spawn_segment_server(0).await; let directory = tempfile::tempdir()?; let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + if head == 3 { + let mut body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); + for seq in 1..=6 { + writeln!(body, "#EXTINF:4,\n{}/{seq}.ts", server.base_url).unwrap(); + } + let mut session = context.session.write().await; + session.apply_origin_manifest(&normal_manifest(&body)).unwrap(); + session.live_window_strip.value = 1; + } install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await; clear_scheduled_prefetch(&context, &policy).await; - context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10); + context.session.write().await.queue_segment_fetch_candidate(head, SegmentFetchPriority::Demand, 10); let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap(); let (entered, resume) = context.segment_cache.pause_startup_write(phase); let task_context = context.clone(); @@ -34,7 +45,7 @@ async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await }); tokio::time::timeout(Duration::from_secs(5), entered).await.unwrap().unwrap(); - let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&1].clone(); + let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&head].clone(); if cancel { task.abort(); assert!(matches!(task.await, Err(error) if error.is_cancelled())); @@ -68,7 +79,7 @@ async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std if phase == StartupWritePhase::AfterPublication { for _ in 0..2 { let metadata = revision.revision().wait_complete(tokio::time::Instant::now()).await?; - assert_eq!(tokio::fs::read(metadata.path).await?, b"body:/1.ts"); + assert_eq!(tokio::fs::read(metadata.path).await?, format!("body:/{head}.ts").as_bytes()); } } Ok(()) diff --git a/backend/hls/src/segment_revision.rs b/backend/hls/src/segment_revision.rs index 7a624b599..8dc9240fb 100644 --- a/backend/hls/src/segment_revision.rs +++ b/backend/hls/src/segment_revision.rs @@ -58,10 +58,23 @@ pub struct SegmentRevision { pub prefix_available: AtomicU64, state: watch::Sender, exposed: AtomicBool, + origin_binding: Mutex>, pub(crate) file_pin: Mutex>, pub(crate) replay: Mutex>, } +/// The origin mapping which produced these bytes, independent of the attempt's +/// mutable Fetching/Ready status. A retained startup map alone is not proof that +/// a revision still belongs to the session's current origin generation. +#[derive(Debug, Clone)] +struct RevisionOriginBinding { + generation: u64, + origin_key: crate::OriginSegmentKey, + cache_key: SegmentCacheKey, + fetch_ref: Option, + encryption: Option, +} + impl fmt::Debug for SegmentRevision { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SegmentRevision") @@ -74,6 +87,32 @@ impl fmt::Debug for SegmentRevision { impl SegmentRevision { pub fn id(&self) -> SegmentRevisionId { self.key.id } + pub(crate) fn was_exposed(&self) -> bool { self.exposed.load(Ordering::Acquire) } + + pub(crate) fn bind_origin(&self, session: &crate::HlsSession) { + if let Some(entry) = session.segments.get(&self.key.proxy_seq) { + *self.origin_binding.lock_unpoisoned() = Some(RevisionOriginBinding { + generation: session.activity.origin_work_generation, + origin_key: entry.origin_key, + cache_key: entry.cache_key.clone(), + fetch_ref: entry.origin_fetch_ref.clone(), + encryption: entry.encryption.clone(), + }); + } + } + + pub(crate) fn origin_binding_is_current(&self, session: &crate::HlsSession) -> bool { + self.key.proxy_session_id() == &session.proxy_session_id + && self.origin_binding.lock_unpoisoned().as_ref().is_some_and(|binding| { + binding.generation == session.activity.origin_work_generation + && session.segments.get(&self.key.proxy_seq).is_some_and(|entry| { + entry.origin_key == binding.origin_key + && entry.cache_key == binding.cache_key + && entry.origin_fetch_ref == binding.fetch_ref + && entry.encryption == binding.encryption + }) + }) + } #[cfg(test)] pub(crate) fn pin_count(&self) -> usize { self.pins.load(Ordering::Acquire) } @@ -117,13 +156,12 @@ impl SegmentRevision { exposed } - /// An optimistic successor is listed before its body arrives. If it fails - /// before receiving any byte, a new attempt may take over its URI. + /// Only client exposure makes failed bytes immutable. Internally buffered + /// prefix bytes do not prevent an optimistic retry from taking over the URI. pub(crate) fn is_replaceable_in(&self, mode: HlsStartupMode) -> bool { mode.publishes_pending_successors() && matches!(*self.state.borrow(), SegmentRevisionState::Failed) && !self.exposed.load(Ordering::Acquire) - && self.prefix_available.load(Ordering::Acquire) == 0 } pub(crate) fn replay_exhausted(&self) { @@ -164,8 +202,9 @@ impl SegmentRevision { } /// Whether this revision may be listed in a startup window. Optimistic - /// startup lists a successor as soon as the origin accepted it; delivery - /// of a successor always waits for its complete body. + /// startup lists a successor as soon as the origin accepted it. The lease's + /// first full request may stream it once replay bytes arrive; subsequent + /// requests wait for its complete body. pub fn is_window_publishable(&self, mode: HlsStartupMode) -> bool { if mode.publishes_pending_successors() { !matches!(*self.state.borrow(), SegmentRevisionState::Failed) @@ -261,6 +300,7 @@ impl SegmentRevisionStore { prefix_available: AtomicU64::new(0), state: watch::channel(SegmentRevisionState::Pending).0, exposed: AtomicBool::new(false), + origin_binding: Mutex::new(None), file_pin: Mutex::new(None), replay: Mutex::new(None), }); diff --git a/backend/hls/src/session.rs b/backend/hls/src/session.rs index 5e0211392..c280f1238 100644 --- a/backend/hls/src/session.rs +++ b/backend/hls/src/session.rs @@ -279,6 +279,13 @@ pub struct HlsSession { pub origin_control: HlsSessionOriginControl, pub render_policy: RenderPolicy, pub startup: Option, + /// None before an origin commit; Some(empty) is an explicitly empty origin window. + pub(crate) origin_window_proxy_seqs: Option>, + pub(crate) origin_window_durations_ms: Vec, + pub(crate) live_window_origin_commit_identity: Option, + pub(crate) live_window_origin_work_generation: Option, + pub live_window_strip: tuliprox_core::model::StripConfig, + pub(crate) live_window_demands: HashMap, pub last_rendered_manifest: Option, manifest_commit_generation: u64, /// Origin manifests committed into this session. @@ -359,6 +366,15 @@ impl HlsSession { last_effective_manifest_host: None, origin_refresh: OriginRefreshState::default(), origin_control: HlsSessionOriginControl::default(), + origin_window_proxy_seqs: None, + origin_window_durations_ms: Vec::new(), + live_window_origin_commit_identity: None, + live_window_origin_work_generation: None, + live_window_strip: tuliprox_core::model::StripConfig { + mode: shared::model::HlsStripMode::Segments, + value: 0, + }, + live_window_demands: HashMap::new(), render_policy: RenderPolicy::default(), startup: None, last_rendered_manifest: None, @@ -428,6 +444,16 @@ impl HlsSession { self.origin_manifest_commits = self.origin_manifest_commits.saturating_add(1); } + pub(crate) fn record_live_window_origin_commit(&mut self, now_ms: u64) { + if self.uses_fast_start_live_window() { + if let Some(identity) = self.next_manifest_commit_identity(now_ms) { + self.manifest_commit_generation = identity.commit_generation(); + self.live_window_origin_commit_identity = Some(identity); + self.live_window_origin_work_generation = Some(self.activity.origin_work_generation); + } + } + } + /// Whether the committed manifest was stored after an origin commit newer /// than `origin_manifest_commits`. Render time alone cannot prove this: /// a segment completion may still re-render the previous origin window. @@ -916,6 +942,12 @@ impl fmt::Debug for HlsSession { .field("last_normal_manifest_commit_identity", &self.last_normal_manifest_commit_identity) .field("published_live_origin_baseline", &self.published_live_origin_baseline) .field("longest_rendered_playlist_duration_ms", &self.longest_rendered_playlist_duration_ms) + .field("origin_window_proxy_seqs", &self.origin_window_proxy_seqs) + .field("origin_window_durations_ms", &self.origin_window_durations_ms) + .field("live_window_origin_commit_identity", &self.live_window_origin_commit_identity) + .field("live_window_origin_work_generation", &self.live_window_origin_work_generation) + .field("live_window_strip", &self.live_window_strip) + .field("live_window_demands", &self.live_window_demands) .field("initial_prefetch_gap_segments", &self.initial_prefetch_gap_segments) .field("segment_prefetch_queue_len", &self.segment_prefetch_queue.len()) .field("active_segment_fetches", &self.active_segment_fetches) diff --git a/backend/hls/src/startup_policy.rs b/backend/hls/src/startup_policy.rs index 364554199..6a12b7f53 100644 --- a/backend/hls/src/startup_policy.rs +++ b/backend/hls/src/startup_policy.rs @@ -20,6 +20,7 @@ pub struct HlsSessionStartup { #[derive(Debug, Clone, Eq, PartialEq)] pub struct HlsManifestRevisions { + pub window_plan: Option, pub mode: HlsStartupMode, pub retained_start_seq: u64, pub revisions: BTreeMap, diff --git a/backend/hls/src/timeline.rs b/backend/hls/src/timeline.rs index 1d3072259..5b9d4f375 100644 --- a/backend/hls/src/timeline.rs +++ b/backend/hls/src/timeline.rs @@ -335,6 +335,7 @@ struct TimelineDraft { origin_version: Option, target_duration: Option, independent_segments: bool, + origin_window_proxy_seqs: Vec, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -383,6 +384,7 @@ impl From<&HlsSession> for TimelineDraft { origin_version: session.origin_version, target_duration: session.target_duration, independent_segments: session.independent_segments, + origin_window_proxy_seqs: session.live_origin_sequences().to_vec(), } } } @@ -502,6 +504,12 @@ impl HlsSession { self.origin_version = draft.origin_version; self.target_duration = draft.target_duration; self.independent_segments = draft.independent_segments; + self.origin_window_durations_ms = draft + .origin_window_proxy_seqs + .iter() + .filter_map(|seq| self.segments.get(seq).map(|entry| entry.duration_ms)) + .collect(); + self.origin_window_proxy_seqs = Some(draft.origin_window_proxy_seqs); } /// Captures the contiguous timeline beginning at the lease's first advertised sequence. @@ -591,6 +599,7 @@ impl TimelineDraft { mode: TimelineApplyMode, ) -> Result { let candidate_plan = self.candidate_timeline_plan(manifest, effective_host_id, mode)?; + self.origin_window_proxy_seqs.clear(); if candidate_plan.segments.is_empty() { return Ok(TimelineApplyResult::default()); } @@ -644,6 +653,7 @@ impl TimelineDraft { self.publishable_origin_head_proxy_seq = handoff_publishable_head_proxy_seq.or(manifest_head_proxy_seq); self.publishable_origin_tail_proxy_seq = manifest_tail_proxy_seq; + self.origin_window_proxy_seqs.clone_from(&result.segment_proxy_seqs); Ok(result) } diff --git a/docs/src/configuration/shared-hls-configuration.md b/docs/src/configuration/shared-hls-configuration.md index d963f259d..db25534cc 100644 --- a/docs/src/configuration/shared-hls-configuration.md +++ b/docs/src/configuration/shared-hls-configuration.md @@ -268,6 +268,24 @@ Changing `rewrite_secret` is also disruptive for public HLS identifiers. Existin READY reserve, cache delivery and repair behavior. `first_ready` lowers initial admission to one processed start segment. Every listed successor must already have a successful HTTP 200 response and a nonempty body prefix. +For untagged, rolling live playlists, each new access lease in `first_ready`, `progressive` or `optimistic` +selects its own three-segment entry window from the newest end of the current origin playlist. This also applies +when joining an already running shared session. `hls_cache.strip` is evaluated before selecting that window: +segment mode counts backwards from the live edge; seconds mode sums the actual `EXTINF` durations backwards. +The holdback is capped so at least three segments remain when the origin provides three or more. For origin +segments `0–5`, strip `0` starts at `3,4,5`; strip `1` starts at `2,3,4`. These three segments are fetched in +chronological order with `RenderWindow` priority. Held-back newer segments are optional reserve prefetch. + +After activation, strip no longer applies to that lease. Its window grows forwards up to six entries, without +prepending segments before its previously advertised first sequence: `2,3,4` becomes `2–5`, then `2–6`, then +`2–7`. Needed successors remain required fetch work even when reserve prefetch is disabled. Active slides retain +contiguous cached predecessors from the lease's previous window when needed to preserve three target durations. +A second lease independently selects its own three-segment entry. An idle re-entry also selects an entry +window, while retaining that lease's progressive GET allowance and monotonic media sequence. A change that +would move its sequence backwards keeps its previous valid view or returns the existing temporary error. +The planned view is never stripped again during lease materialization. These rules do not change conservative, +archive, ENDLIST, EVENT, VOD or other protected/transient playlist handling, or short-origin qualification. + ```yaml reverse_proxy: hls_cache: @@ -280,17 +298,30 @@ reverse_proxy: max_deferred_repairs: 32 ``` -`progressive` lets the first full segment request consume the existing shared origin fill before origin EOF. +`progressive` lets each access lease's first authorized full segment GET consume the existing shared origin +fill before origin EOF, regardless of its position in the published playlist. A full request has no Range header +or uses `Range: bytes=0-`; HEAD and partial ranges wait for complete metadata/bytes and do not consume this +opportunity. The first full request reserves the opportunity before waiting, even if its revision is already +complete. Concurrent or subsequent requests, including retries, wait for complete bytes. A cancelled first GET +does not re-arm the opportunity. A new access lease has its own opportunity; playlist refreshes and seeks do not +reset an existing lease's allowance. The complete session delivers immutable raw TS revisions, including subsequent requests, retries and ranges. Repair and the corruption watchdog continue on the processed cache copy. Raw delivery does not correct source timestamp or transport-stream defects; prefer `first_ready` when a source needs repair. `optimistic` behaves like `progressive` and publishes the initial playlist earlier. The head segment needs a -body prefix, while listed successors only need an accepted HTTP 200 origin response. Successor fetches start -together with the head, within the configured per-session concurrency. A client request for a successor waits -for its complete body. If the origin fails a successor after it was listed, only that segment fails for the -client; already published media never changes. Replay or disk fallback switches the session to `first_ready` -exactly like `progressive`. +body prefix, while listed successors only need an accepted HTTP 200 origin response. This head requirement +controls playlist publication; it does not determine which segment the player requests first. Successor fetches start +together with the head, within the configured per-session concurrency. The lease's first full request may select +any published successor. It waits for a replay prefix, then streams before EOF; without replay capacity it waits +for the complete revision instead. Other requests wait for complete bodies. If the origin fails after a revision +has been exposed, the body errors and recovery requires a new playback entry; bytes already exposed are never +silently replaced. A failed unexposed revision can recover through its successful retry. Existing replay and disk +fallback rules remain in effect; successors do not change the head's session policy. + +A revision is considered exposed when a progressive response is prepared, before the first body chunk is sent. +This conservative boundary applies across leases sharing that revision, even if the first client disconnects +before reading any bytes. Listing a revision in a manifest alone does not expose it. Fast startup is limited to live, clear MPEG-TS objects without MAP or origin byte ranges. Archive, encrypted, fragmented-MP4 and transient sessions use their existing path. Before publication an incompatible source can fall back to that path. diff --git a/docs/src/configuration/shared-hls-troubleshooting.md b/docs/src/configuration/shared-hls-troubleshooting.md index 6b928aab9..9aa3372d9 100644 --- a/docs/src/configuration/shared-hls-troubleshooting.md +++ b/docs/src/configuration/shared-hls-troubleshooting.md @@ -158,12 +158,27 @@ Fix: A `503` with `Retry-After` can be normal during short origin/cache waits, especially near startup or when a requested segment is known but not ready yet. +An optimistic segment whose origin download failed before any bytes were exposed to a client stays temporarily +unavailable while its retry chain is active. Its existing URI returns `503` with `Retry-After` during that wait, +instead of a permanent `404`. After a successful retry, the same lease and URI serve the complete replacement; +the player does not need to refresh the playlist first. Internally buffered prefix bytes do not count as exposure. +If a failed revision was already exposed, Tuliprox ends only the affected lease and its old routes return `404`. +A new playback entry can use the shared session. Completed revisions and already exposed bytes are never replaced. +The first authorized full segment GET of each lease can claim progressive delivery, including for a published +successor. HEAD and partial ranges do not consume that opportunity; subsequent GETs wait for complete bytes. +See [startup modes](./shared-hls-configuration.md#optional-startup-modes) for the shared policy. + +The degraded-origin admission check still applies: a retryable manifest failure disables accelerated admission, +and an unchanged successful manifest does not clear that condition. Media progress is required to clear it. + Investigate when it repeats for too long: | Log signal | Interpretation | | :--- | :--- | | `HLS segment demand fetch skipped by backpressure` | The session or global fetch limit is saturated. | | `HLS segment temporary failure counted` | The origin segment fetch is failing temporarily. | +| `HLS revision recovery` | `replacement_bound` repairs an unexposed failed binding; `reentry_required` ends that exact lease. Includes anonymized identity, revision, sequence, generation, prefix size, and exposure. | +| `HLS playlist wait` | A changed wait reason: `fresh_render_required`, `missing_revision_or_head_readiness`, or `admission_rejected`. Includes head readiness and degraded-origin state; repeated identical polls are not logged. | | `HLS manifest marked fresh-commit required after hard fetch failure` | Manifest refresh had a hard failure and needs a fresh commit. | | `HLS terminal manifest failed closed` | A typed lease-local terminal decision could not safely serve live or terminal media. The reason identifies state, bundle, deadline, capacity, or retry failure. | | `HLS terminal commit retry owner failed closed` | The bounded autonomous commit worker reached a typed terminal failure; inspect the reason and preceding recovery/bundle events. |