diff --git a/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs index 552348c2e..2b6837e98 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs @@ -1,107 +1,93 @@ use super::*; -#[tokio::test] -async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() { +async fn exhausted_fixture() -> (terminal_tail::PublicationLateFixture, String) { let fixture = terminal_tail::publication_late_fixture().await; - let now_ms = super::super::current_time_millis(); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("published live lease"); - let snapshot = lease.last_manifest_snapshot.expect("published snapshot"); - { - let mut session = fixture.session.write().await; - session.segments.clear(); - session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; - } - let requests_before = fixture.origin.manifest_request_count(); let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - assert!(!response.headers().contains_key(header::RETRY_AFTER)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); - assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1))); - assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000))); - assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n")); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); - assert!(!body.contains("/hls/shared/live/")); - let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::>(); - assert_eq!(media_uris.len(), 12); - let (first, second) = tokio::join!( - get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), - get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), - ); - for response in [first, second] { - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body(response).await, body.as_bytes()); - } - let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; - assert_eq!(media.status(), StatusCode::OK); - assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t"); - let bytes = response_body(media).await; - assert!(!bytes.is_empty()); - let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await; - assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response_body(range).await, bytes.slice(..188)); - let router = - crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state)); - let head = router - .oneshot( - Request::builder() - .method(Method::HEAD) - .uri(media_uris[0]) - .extension(ConnectInfo(test_addr())) - .body(Body::empty()) - .expect("HEAD request"), - ) + let previous = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + let manager = &fixture.app_state.hls.proxy; + let now_ms = super::super::current_time_millis(); + let lease = manager + .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) .await - .expect("HEAD response"); - assert_eq!(head.status(), StatusCode::OK); - assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string()); - assert!(response_body(head).await.is_empty()); - fixture.app_state.app_config.custom_stream_response.store(None); - let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await; - assert_eq!(replay.status(), StatusCode::OK); - assert_eq!(response_body(replay).await, bytes); - let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease"); - assert_eq!( - custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(), - StatusCode::NOT_FOUND - ); - let wrong_index = media_uris[0].replace("/0.ts", "/12.ts"); - assert_eq!( - custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(), - StatusCode::NOT_FOUND - ); - let late = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot( + .unwrap(); + let identity = lease.media_identity().unwrap(); + let first = lease.last_manifest_snapshot.as_ref().unwrap().first_proxy_seq; + // Virtual established playback: the media clock begins with delivered bytes, + // and has consumed the complete finite READY horizon before this outage. + let started_at_ms = now_ms - 120_000; + let token = manager + .record_access_lease_segment_request_started_if_identity_matches( &fixture.access_lease_id, &fixture.proxy_session_id, - super::super::current_time_millis(), + identity, + first, + started_at_ms, ) .await - .expect("terminal lease"); - assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); - assert_eq!(late.playback_cursor, lease.playback_cursor); - assert!(!fixture.session.read().await.has_terminal_tail_protections()); - assert_eq!(fixture.origin.manifest_request_count(), requests_before); + .unwrap(); + assert_eq!( + manager + .mark_delivered_media_for_lease( + &fixture.session, + &fixture.access_lease_id, + &fixture.proxy_session_id, + identity, + Some(token), + started_at_ms + ) + .await, + tuliprox_hls::api::HlsMediaActivityCommitOutcome::Committed + ); + let mut session = fixture.session.write().await; + session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; + session.origin_refresh.mark_started(now_ms); + drop(session); + (fixture, previous) } #[tokio::test] -async fn late_custom_response_without_configured_asset_retains_error_response() { - let fixture = terminal_tail::publication_late_fixture().await; +async fn expired_splice_deadline_serves_exact_last_live_uri_and_immutable_terminal_tail() { + let (fixture, previous) = exhausted_fixture().await; + let last_live = previous.lines().rfind(|line| !line.is_empty() && !line.starts_with('#')).unwrap(); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3); + let body = loop { + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + if response.status() == StatusCode::OK { + let body = String::from_utf8(response_body(response).await.to_vec()).unwrap(); + if body.contains("#EXT-X-ENDLIST") { + break body; + } + } + assert!(tokio::time::Instant::now() < deadline, "terminal owner must finish"); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + }; + assert!(!body.contains("/cvs/hls/")); + let media = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::>(); + let first_csv = media.iter().position(|uri| uri.contains("/terminal/")).unwrap(); + assert!(first_csv > 0); + assert_eq!(media[first_csv - 1], last_live); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + let live = get_response(Arc::clone(&fixture.app_state), last_live, None).await; + assert_eq!(live.status(), StatusCode::OK); + assert!(!response_body(live).await.is_empty()); + let csv = get_response(Arc::clone(&fixture.app_state), media[first_csv], None).await; + assert_eq!(csv.status(), StatusCode::OK); + assert!(!response_body(csv).await.is_empty()); fixture.app_state.app_config.custom_stream_response.store(None); - { - let mut session = fixture.session.write().await; - session.segments.clear(); - session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; - } + let (a, b) = tokio::join!( + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None), + get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None) + ); + assert_eq!(response_body(a).await, body.as_bytes()); + assert_eq!(response_body(b).await, body.as_bytes()); + assert!(fixture.session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn missing_live_base_fails_explicitly_instead_of_publishing_csv_only() { + let (fixture, _) = exhausted_fixture().await; + fixture.session.write().await.segments.clear(); let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(response_body(response).await.is_empty()); @@ -115,99 +101,18 @@ async fn late_custom_response_without_configured_asset_retains_error_response() super::super::current_time_millis(), ) .await - .expect("terminal unavailable lease"); + .unwrap(); assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); } #[tokio::test] -async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() { - let (fixture, media_uri) = committed_late_http_fixture().await; - for (method, range, status) in [ - (Method::GET, None, StatusCode::OK), - (Method::HEAD, None, StatusCode::OK), - (Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT), - ] { - let now_ms = super::super::current_time_millis(); - let previous_access_ms = now_ms.saturating_sub(50_000); - assert!(fixture - .app_state - .hls - .proxy - .activate_access_lease( - &fixture.access_lease_id, - &fixture.proxy_session_id, - previous_access_ms, - HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - let before = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("valid late lease"); - let response = - custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await; - assert_eq!(response.status(), status); - let after = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot( - &fixture.access_lease_id, - &fixture.proxy_session_id, - super::super::current_time_millis(), - ) - .await - .expect("renewed late lease"); - assert!(after.last_seen_at_ms >= now_ms); - assert!(after.valid_until_ms > before.valid_until_ms); - assert!(after.active_until_ms > before.active_until_ms); - assert_eq!(after.playback_mode, before.playback_mode); - assert_eq!(after.playback_cursor, before.playback_cursor); - assert_eq!(after.admission_generation, before.admission_generation); - } -} - -#[tokio::test] -async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() { - let (fixture, media_uri) = committed_late_http_fixture().await; - let now_ms = super::super::current_time_millis(); - assert!(fixture - .app_state - .hls - .proxy - .activate_access_lease( - &fixture.access_lease_id, - &fixture.proxy_session_id, - now_ms - 50_000, - HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - let before = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("valid late lease"); - let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint"); - for (uri, range, expected) in [ - (media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND), - (media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND), - (media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND), - (media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE), - ] { - let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await; - assert_eq!(response.status(), expected); - } - fixture.app_state.app_config.api_proxy.store(None); - let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await; - assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status()); - let after = fixture +async fn missing_csv_asset_keeps_the_explicit_error_response() { + let (fixture, _) = exhausted_fixture().await; + fixture.app_state.app_config.custom_stream_response.store(None); + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response_body(response).await.is_empty()); + let lease = fixture .app_state .hls .proxy @@ -217,43 +122,6 @@ async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() super::super::current_time_millis(), ) .await - .expect("unchanged late lease"); - assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms); - assert_eq!(after.valid_until_ms, before.valid_until_ms); - assert_eq!(after.active_until_ms, before.active_until_ms); -} - -async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) { - let fixture = terminal_tail::publication_late_fixture().await; - { - let mut session = fixture.session.write().await; - session.segments.clear(); - session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict; - } - let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest"); - let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned(); - (fixture, media_uri) -} - -async fn custom_media_response(app_state: Arc, uri: &str, range: Option<&str>) -> Response { - custom_media_response_with_method(app_state, uri, range, Method::GET).await -} - -async fn custom_media_response_with_method( - app_state: Arc, - uri: &str, - range: Option<&str>, - method: Method, -) -> Response { - let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state); - let mut request = Request::builder().method(method).uri(uri); - if let Some(range) = range { - request = request.header(header::RANGE, range); - } - router - .oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request")) - .await - .expect("custom media response") + .unwrap(); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); } diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs index b2192e10f..d898ae88a 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs @@ -141,20 +141,25 @@ async fn wait_for_shared_first_ready_refresh( fixture: &ShortWindowFixture, session: &HlsSessionHandle, previous_requests: usize, + expected_generation: u64, ) { tokio::time::timeout(Duration::from_secs(5), async { loop { - if fixture.origin.manifest_request_count() > previous_requests - && !session.read().await.origin_refresh.in_flight - { + let refresh_in_flight = { + let session = session.read().await; + // Later leases may start another ordinary freshness fetch after a fast + // response; they must still share the completed qualification. + assert_eq!(session.origin_control.acceptance_generation.0, expected_generation); + session.origin_refresh.in_flight + }; + if fixture.origin.manifest_request_count() > previous_requests && !refresh_in_flight { break; } tokio::time::sleep(Duration::from_millis(5)).await; } }) .await - .expect("additional leases complete one shared freshness request"); - assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1); + .expect("additional leases complete freshness checks without requalification"); } async fn assert_first_ready_lease_media( @@ -217,7 +222,7 @@ async fn first_ready_leases_share_qualification_while_first_segment_is_loading() let fixture = Arc::clone(&fixture); leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await }); } - wait_for_shared_first_ready_refresh(&fixture, &session, 3).await; + wait_for_shared_first_ready_refresh(&fixture, &session, 3, generation.0).await; assert!(session.read().await.last_rendered_manifest.is_none()); assert_eq!(session.read().await.origin_control.acceptance_generation, generation); gate.close(); @@ -316,7 +321,7 @@ async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loa let fixture = Arc::clone(&fixture); leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await }); } - wait_for_shared_first_ready_refresh(&fixture, &session, 5).await; + wait_for_shared_first_ready_refresh(&fixture, &session, 5, 2).await; assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2); assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit)); gate.close(); diff --git a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs index 46d250ef4..ee35397d9 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs @@ -153,6 +153,16 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) { "{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}", binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0 ); + { + let mut leases = app.hls.proxy.access_leases().write().await; + let mut active = leases.remove_access_lease(&HlsAccessLeaseId(lease.to_string())).unwrap(); + let start_seq = active.playback_cursor.first_segment_started_proxy_seq.unwrap(); + let state = session.read().await; + let start = state.segments[&start_seq].media_start_ms; + let horizon = state.segments.values().map(|entry| entry.media_start_ms + entry.duration_ms).max().unwrap(); + active.playback_cursor.first_segment_started_at_ms = Some(current_time_millis() - (horizon - start - 8_000)); + leases.prepare_access_lease(active); + } session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure; let directive = match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id) diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs index e259a478b..9a9c832c5 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs @@ -694,6 +694,10 @@ pub(in crate::api::endpoints::hls_api::tests) async fn prepare_terminal_cutover_ (asset_buffer, asset) } +#[allow( + clippy::too_many_lines, + reason = "One encrypted playback fixture includes real delivery, elapsed media time and prepared terminal bytes." +)] pub(in crate::api::endpoints::hls_api::tests) async fn prepared_terminal_cutover_fixture( ) -> PreparedTerminalCutoverFixture { let temp_dir = tempfile::tempdir().expect("terminal cutover tempdir"); @@ -778,6 +782,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn prepared_terminal_cutover assert_eq!(response.status(), StatusCode::OK); assert!(!response_body(response).await.is_empty()); } + { + let mut leases = app_state.hls.proxy.access_leases().write().await; + let mut lease = leases.remove_access_lease(&lease_id).unwrap(); + lease.playback_cursor.first_segment_started_at_ms = + lease.playback_cursor.first_segment_started_at_ms.map(|at| at - 10_000); + leases.prepare_access_lease(lease); + } PreparedTerminalCutoverFixture { _temp_dir: temp_dir, origin, @@ -923,12 +934,10 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_ ) -> (u64, u64) { let cutover_now_ms = pressured_lease .playback_cursor - .first_segment_completed_at_ms - .expect("measured lease playback start") - // Enter the acquisition window while the live base and its key are still usable. - .saturating_add( - fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms), - ); + .first_segment_started_at_ms + .expect("actual transfer start") + .saturating_add(fixture.base_manifest.playlist_duration_ms) + .saturating_add(3_800); let first = commit_terminal_tail_if_lease_reserve_requires_cutover( &fixture.app_state.hls_ctx(), diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs index 7072a7945..27a79e02a 100644 --- a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs @@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path() } #[tokio::test] -async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() { +async fn commits_prepared_terminal_tail_once_after_complete_reserve_consumption() { let fixture = prepared_terminal_cutover_fixture().await; let directive = terminal_cutover_acceptance_directive(&fixture).await; let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await; @@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhaus } #[tokio::test] -async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() { +async fn warm_fmp4_at_exhaustion_preserves_the_explicit_incompatible_media_error() { let temp_dir = tempfile::tempdir().expect("tempdir"); let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); enable_channel_unavailable_custom_response(&app_state); @@ -143,39 +143,41 @@ async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splic }; publish_test_manifest_and_exhaust_configured_acceptance(&app_state, &proxy_session_id, &lease_id, snapshot, now_ms) .await; + let manager = &app_state.hls.proxy; + let session = manager.sessions().get_by_proxy_session_id(&proxy_session_id).await.unwrap(); + let identity = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .unwrap() + .media_identity() + .unwrap(); + let started_at = now_ms - 120_000; + let token = manager + .record_access_lease_segment_request_started_if_identity_matches( + &lease_id, + &proxy_session_id, + identity, + base_proxy_seq, + started_at, + ) + .await + .unwrap(); + let _ = manager + .mark_delivered_media_for_lease(&session, &lease_id, &proxy_session_id, identity, Some(token), started_at) + .await; let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest"); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); - assert!(body.contains("/cvs/hls/")); - assert!(!body.contains("#EXT-X-MAP")); - assert!(!body.contains("/hls/shared/live/")); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response_body(response).await.is_empty()); let lease = app_state .hls .proxy .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) .await - .expect("late custom lease snapshot"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); - let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - assert_eq!(repeated.status(), StatusCode::OK); - assert_eq!(response_body(repeated).await, body.as_bytes()); - let repeated_lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) - .await - .expect("stable terminal custom lease"); - assert_eq!(repeated_lease.playback_mode, lease.playback_mode); - let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); - let terminal_response = get_response(app_state, &terminal_uri, None).await; - assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); - assert!(!terminal_response.headers().contains_key(header::LOCATION)); - assert!(response_body(terminal_response).await.is_empty()); + .unwrap(); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); } #[tokio::test] diff --git a/backend/app/tests/hls_manifest_window.rs b/backend/app/tests/hls_manifest_window.rs deleted file mode 100644 index bf5827978..000000000 --- a/backend/app/tests/hls_manifest_window.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Real process regression against the existing local Postman hls-window fixture. -//! -//! Start its simulation with a fixed port and an NDJSON log: -//! `/Users/philippmielke/.local/bin/postman simulation run postman/simulations/hls-window.sim.yaml --port 4926 --no-history --output ndjson` -//! -//! Run through `./bin/test.sh -p tuliprox --test hls_manifest_window -- --ignored`, -//! setting `TULIPROX_HLS_WINDOW_SIMULATION_ORIGIN=http://127.0.0.1:4926/origin` -//! and `TULIPROX_HLS_WINDOW_EVIDENCE_DIR` to an absolute output directory. -//! The member serves real HLS metadata; a loopback TS origin gates new bytes. -//! Only the URL-free client protocol is suitable for sharing; keep raw logs local. -use std::{ - io::Write as _, - process::Stdio, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - time::Duration, -}; -use tokio::{ - io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, - net::TcpListener, - sync::watch, -}; - -const BODY: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - -fn media_url(base: &str, playlist: &str) -> String { - let uri = playlist.lines().find(|line| !line.is_empty() && !line.starts_with('#')).unwrap(); - reqwest::Url::parse(base).unwrap().join(uri).unwrap().to_string() -} - -fn sequence(body: &str, tag: &str) -> u64 { - body.lines().find_map(|line| line.strip_prefix(tag)).unwrap_or("0").parse().unwrap() -} - -#[tokio::test] -#[allow( - clippy::too_many_lines, - reason = "Keep the real process, streaming gate and HTTP assertions in one isolated scenario." -)] -#[ignore = "requires local Postman hls-window simulation; see module documentation"] -async fn live_window_http_republishes_same_lease_while_successor_is_gated() { - let simulation = std::env::var("TULIPROX_HLS_WINDOW_SIMULATION_ORIGIN").expect("Postman origin URL"); - let evidence = - std::path::PathBuf::from(std::env::var("TULIPROX_HLS_WINDOW_EVIDENCE_DIR").expect("evidence directory")); - std::fs::create_dir_all(&evidence).unwrap(); - let mut protocol = std::fs::File::create(evidence.join("client.ndjson")).unwrap(); - let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap(); - for mode in ["first_ready", "progressive", "optimistic"] { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let segment_origin = format!("http://{}", listener.local_addr().unwrap()); - let configure = |first| { - client - .post(format!("{simulation}/configure")) - .json(&serde_json::json!({"firstSequence": first, "segmentOrigin": segment_origin})) - }; - assert_eq!(configure(0).send().await.unwrap().status(), 204); - let catalog = format!( - "#EXTM3U\n#EXTINF:-1 tvg-id=\"window\" group-title=\"Test\",Window Channel\n{simulation}/index.m3u8\n" - ); - let (release, released) = watch::channel(false); - let attempts = Arc::new(AtomicUsize::new(0)); - let counted = attempts.clone(); - let origin = tokio::spawn(async move { - let mut children = tokio::task::JoinSet::new(); - loop { - tokio::select! { - accepted = listener.accept() => { - let (socket, _) = accepted.unwrap(); - let catalog = catalog.clone(); - let mut released = released.clone(); - let attempts = counted.clone(); - children.spawn(async move { - let mut reader = BufReader::new(socket); - let mut request = String::new(); reader.read_line(&mut request).await.unwrap(); - let path = request.split_whitespace().nth(1).unwrap().to_string(); - loop { - let mut line = String::new(); reader.read_line(&mut line).await.unwrap(); - if line == "\r\n" || line.is_empty() { break; } - } - let mut socket = reader.into_inner(); - let body = if path == "/catalog.m3u" { catalog.as_bytes() } else { BODY }; - let seq = path.strip_prefix('/').and_then(|path| path.strip_suffix(".ts")) - .and_then(|seq| seq.parse::().ok()); - if seq.is_some_and(|seq| seq >= 6) { - attempts.fetch_add(1, Ordering::AcqRel); - while !*released.borrow_and_update() { - if released.changed().await.is_err() { return; } - } - } - let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - if socket.write_all(headers.as_bytes()).await.is_ok() { let _ = socket.write_all(body).await; } - }); - } - _ = children.join_next(), if !children.is_empty() => {} - } - } - }); - let directory = tempfile::tempdir().unwrap(); - let root = directory.path(); - for name in ["web", "storage", "backup"] { - std::fs::create_dir(root.join(name)).unwrap(); - } - let bound = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = bound.local_addr().unwrap().port(); - drop(bound); - let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n hls_cache:\n cache_path: '{}'\n strip:\n mode: segments\n value: 0\n origin_segment_timeout_ms: 10000\n max_concurrent_segment_fetches_per_session: 6\n initial_manifest_wait_timeout_secs: 3\n startup:\n mode: {mode}\n segment_repair:\n max_level: off\n", root.join("web").display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display()); - let sources = format!("inputs:\n - name: window-origin\n type: m3u\n url: {segment_origin}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [window-origin]\n targets:\n - name: window\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n"); - let users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Window test\nuser:\n - target: window\n credentials:\n - username: viewer\n password: test-password\n proxy: reverse\n server: local\n status: Active\n max_connections: 1\n"); - for (name, body) in [("config.yml", config), ("source.yml", sources), ("api-proxy.yml", users)] { - std::fs::write(root.join(name), body).unwrap(); - } - let log_file = evidence.join(format!("sut-{mode}.log")); - let log = std::fs::File::create(&log_file).unwrap(); - let mut process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox")) - .args(["-s", "-c", "config.yml", "-i", "source.yml", "-a", "api-proxy.yml"]) - .current_dir(root) - .env("RUST_LOG", "debug") - .stdout(Stdio::from(log.try_clone().unwrap())) - .stderr(Stdio::from(log)) - .kill_on_drop(true) - .spawn() - .unwrap(); - let base = format!("http://127.0.0.1:{port}"); - let deadline = tokio::time::Instant::now() + Duration::from_secs(30); - let entry = loop { - assert!( - process.try_wait().unwrap().is_none(), - "SUT exited: {}", - std::fs::read_to_string(&log_file).unwrap() - ); - if let Ok(response) = client.get(format!("{base}/m3u?username=viewer&password=test-password")).send().await - { - if let Ok(body) = response.text().await { - if let Some(url) = body.lines().find(|line| line.starts_with("http")) { - break url.to_owned(); - } - } - } - assert!(tokio::time::Instant::now() < deadline, "catalog startup timeout"); - tokio::time::sleep(Duration::from_millis(25)).await; - }; - let master = client.get(&entry).send().await.unwrap().error_for_status().unwrap().text().await.unwrap(); - let variant = media_url(&entry, &master); - let initial = client.get(&variant).send().await.unwrap(); - assert_eq!(initial.status(), 200); - let previous = initial.text().await.unwrap(); - writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"initial","status":200,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap(); - let segment = client.get(media_url(&variant, &previous)).send().await.unwrap(); - assert_eq!(segment.status(), 200); - assert!(!segment.bytes().await.unwrap().is_empty()); - assert_eq!(configure(6).send().await.unwrap().status(), 204); - let deadline = tokio::time::Instant::now() + Duration::from_secs(8); - while attempts.load(Ordering::Acquire) == 0 { - let response = client.get(&variant).send().await.unwrap(); - assert_eq!(response.status(), 200, "{mode}: waiting for origin refresh"); - assert_eq!(response.text().await.unwrap(), previous); - writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"refresh","status":200,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap(); - assert!(tokio::time::Instant::now() < deadline, "successor origin request was never made"); - tokio::time::sleep(Duration::from_millis(25)).await; - } - for poll in 0..20 { - let response = client.get(&variant).send().await.unwrap(); - let status = response.status().as_u16(); - assert_eq!(status, 200, "{mode} held poll {poll}"); - assert!(response.headers().get("retry-after").is_none()); - assert_eq!(response.text().await.unwrap(), previous); - writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"held","poll":poll,"status":status,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:"),"successorOriginRequests":attempts.load(Ordering::Acquire)})).unwrap(); - } - let retained_responses = std::fs::read_to_string(&log_file) - .unwrap() - .lines() - .filter(|line| line.contains("HLS live window retained:")) - .count(); - assert!(retained_responses >= 20, "{mode}: held HTTP requests must exercise the retained-window branch"); - release.send(true).unwrap(); - let deadline = tokio::time::Instant::now() + Duration::from_secs(8); - loop { - let response = client.get(&variant).send().await.unwrap(); - assert_eq!(response.status(), 200); - let next = response.text().await.unwrap(); - if sequence(&next, "#EXT-X-MEDIA-SEQUENCE:") > sequence(&previous, "#EXT-X-MEDIA-SEQUENCE:") { - assert!( - sequence(&next, "#EXT-X-DISCONTINUITY-SEQUENCE:") - >= sequence(&previous, "#EXT-X-DISCONTINUITY-SEQUENCE:") - ); - writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"advanced","status":200,"mediaSequence":sequence(&next,"#EXT-X-MEDIA-SEQUENCE:"),"successorOriginRequests":attempts.load(Ordering::Acquire),"retainedResponses":retained_responses})).unwrap(); - break; - } - writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"released","status":200,"mediaSequence":sequence(&next,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap(); - assert!(tokio::time::Instant::now() < deadline, "successor did not publish"); - tokio::time::sleep(Duration::from_millis(25)).await; - } - process.kill().await.unwrap(); - origin.abort(); - } -} diff --git a/backend/dvr/src/recording/recording_worker.rs b/backend/dvr/src/recording/recording_worker.rs index 9e3c909e6..d2cea032a 100644 --- a/backend/dvr/src/recording/recording_worker.rs +++ b/backend/dvr/src/recording/recording_worker.rs @@ -991,12 +991,8 @@ mod tests { Ok::<_, std::io::Error>(()) }); let wrapper = directory.path().join("ffmpeg-with-proxy"); - // Explicitly remove proxy exclusions so localhost still exercises the bypass. - tokio::fs::write( - &wrapper, - "#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n", - ) - .await?; + // Keep the listener exclusions supplied by the worker while adding an unusable proxy. + tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?; fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?; let mut recording = make_recording(chrono::Utc::now().timestamp(), 60); recording.url = url; diff --git a/backend/hls/src/availability/mod.rs b/backend/hls/src/availability/mod.rs index e92d13090..242c29f9c 100644 --- a/backend/hls/src/availability/mod.rs +++ b/backend/hls/src/availability/mod.rs @@ -42,7 +42,6 @@ pub use self::{ }; use std::future::Future; -pub const HLS_PLAYBACK_RATE_GUARD_MILLI: u16 = 1_050; const HLS_AVAILABILITY_STATE_ACCESS_ATTEMPTS: usize = 3; const HLS_AVAILABILITY_REEVALUATION_MAX_ATTEMPTS: u8 = 64; const HLS_AVAILABILITY_REEVALUATION_DEADLINE_MS: u64 = 2_000; @@ -169,13 +168,18 @@ pub async fn hls_startup_admission_allows_snapshot( snapshot: &HlsLeaseManifestSnapshot, now_ms: u64, ) -> bool { - let (ready_timeline, origin_state, observed_latency) = { + let (ready_timeline, origin_state, observed_latency, target_duration_ms) = { let session = session.read().await; let degraded = session.origin_control.path_condition.is_degraded(); ( session.ready_timeline_snapshot(snapshot.first_proxy_seq, now_ms), if degraded { HlsStartupAdmissionOriginState::Degraded } else { HlsStartupAdmissionOriginState::Healthy }, session.origin_control.recovery_samples.latency_snapshot(), + session + .origin_control + .target_duration_snapshot_ms + .or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000))) + .unwrap_or(snapshot.target_duration_ms), ) }; let workload = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(); @@ -190,7 +194,7 @@ pub async fn hls_startup_admission_allows_snapshot( origin_state, recovery_trigger_budget: recovery_trigger_budget( pressure_policy, - snapshot.target_duration_ms, + target_duration_ms, workload, observed_latency, ), diff --git a/backend/hls/src/availability/recovery_pressure.rs b/backend/hls/src/availability/recovery_pressure.rs index 958e840c3..2becd201f 100644 --- a/backend/hls/src/availability/recovery_pressure.rs +++ b/backend/hls/src/availability/recovery_pressure.rs @@ -5,7 +5,7 @@ //! timing seed the acceptance episode starts from. Committed back onto the //! session under a CAS loop, so a concurrent refresh cannot lose the decision. -use super::{hls_recovery_timing_policy, HLS_PLAYBACK_RATE_GUARD_MILLI}; +use super::hls_recovery_timing_policy; use crate::{ hls_ctx::HlsCtx, lease::HlsAccessLeaseStore, @@ -207,13 +207,12 @@ pub(super) fn evaluate_and_commit_session_recovery_pressure( now_ms, ); let recovery_trigger_budget = - recovery_trigger_budget(policy, manifest.target_duration_ms, workload, observed_latency); + recovery_trigger_budget(policy, publication_target_duration_ms, workload, observed_latency); let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { manifest, cursor: &lease.playback_cursor, ready_timeline: &ready_timeline, now_ms, - playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget, origin_path_degraded: effective_origin_degraded, recovery_committed: false, @@ -227,7 +226,7 @@ pub(super) fn evaluate_and_commit_session_recovery_pressure( reserve, cursor: lease.playback_cursor.clone(), workload, - target_duration_ms: manifest.target_duration_ms, + target_duration_ms: publication_target_duration_ms, latest_safe_terminal_commit_at: cutover_timing.latest_safe_terminal_commit_at, recovery_boundary_slack_ms: HlsRecoveryBoundarySlackMs::from_reserve_and_boundary( reserve.guaranteed_reserve_ms, diff --git a/backend/hls/src/availability/terminal_cutover.rs b/backend/hls/src/availability/terminal_cutover.rs index 095574e92..ec3de8735 100644 --- a/backend/hls/src/availability/terminal_cutover.rs +++ b/backend/hls/src/availability/terminal_cutover.rs @@ -8,12 +8,10 @@ use super::{ recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget}, terminal_pending_owner::{ - commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure, commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner, terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback, }, HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution, - HLS_PLAYBACK_RATE_GUARD_MILLI, }; use crate::{ hls_ctx::HlsCtx, @@ -85,7 +83,6 @@ struct HlsLeaseTerminalDecisionContext<'a> { state: &'a HlsLeaseCutoverStateSnapshot, evaluation: HlsLeaseTerminalCutoverEvaluation, now_ms: u64, - purpose: HlsTerminalDecisionPurpose, } impl<'a> HlsLeaseTerminalDecisionContext<'a> { @@ -124,13 +121,12 @@ fn evaluate_lease_terminal_cutover( state.session_target_duration, ); let recovery_trigger_budget = - recovery_trigger_budget(pressure_policy, manifest.target_duration_ms, workload, state.observed_latency); + recovery_trigger_budget(pressure_policy, state.target_duration_ms, workload, state.observed_latency); let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { manifest, cursor: &lease.playback_cursor, ready_timeline: &state.ready_timeline, now_ms, - playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget, origin_path_degraded, recovery_committed: false, @@ -163,39 +159,14 @@ fn evaluate_lease_terminal_cutover( } } -async fn resolve_terminal_cutover_before_commit_window( +fn resolve_terminal_cutover_before_commit_window( context: &HlsLeaseTerminalDecisionContext<'_>, ) -> HlsDetailedTerminalResolution { if !context.evaluation.origin_path_degraded { return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed); } - if matches!(context.purpose, HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback) { - let Some(preparation) = context - .ctx - .hls_proxy - .prepare_access_lease_terminal_unavailable_after_owner_failure(context.preparation_request()) - .await - else { - return HlsDetailedTerminalResolution::with_deadline( - HlsTerminalResolution::Reevaluate, - context.evaluation.safe_deadline, - ); - }; - return HlsDetailedTerminalResolution::with_deadline( - commit_prepared_terminal_unavailable_after_owner_failure( - context.ctx, - context.session, - context.proxy_session_id, - &context.lease.lease_id, - &preparation, - context.now_ms, - HlsTerminalTailCompatibility::TerminalMediaNotReady, - ), - context.evaluation.safe_deadline, - ); - } let Some(live_reserve_deadline) = context.evaluation.safe_deadline else { - return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::Reevaluate); + return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed); }; HlsDetailedTerminalResolution { resolution: HlsTerminalResolution::LiveAllowed, @@ -306,6 +277,12 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed); } let evaluation = evaluate_lease_terminal_cutover(ctx, lease, manifest, &state, now_ms); + log::debug!("HLS terminal reserve decision: proxy_session={} lease={} start_at_ms={:?} position_known={} position_ms={} reserve_ms={} session_target_duration_ms={} snapshot_generation={} may_publish={}", + crate::safe_proxy_session_id(proxy_session_id), crate::safe_hls_access_lease_id(&lease.lease_id), + lease.playback_cursor.first_segment_started_at_ms, evaluation.reserve.playback_position_known, + evaluation.reserve.conservative_playback_position_ms, evaluation.reserve.guaranteed_reserve_ms, + state.target_duration_ms, manifest.snapshot_generation, + evaluation.origin_path_degraded && evaluation.reserve.is_exhausted()); { let mut session = session.write().await; if session.origin_control.progress_generation == state.progress_generation @@ -320,7 +297,7 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai ); } } - let commit_window = evaluation.commit_window; + let may_publish = evaluation.origin_path_degraded && evaluation.reserve.is_exhausted(); let decision_context = HlsLeaseTerminalDecisionContext { ctx, session, @@ -330,12 +307,11 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai state: &state, evaluation, now_ms, - purpose, }; - if commit_window == HlsTerminalCommitWindow::NotDue { - resolve_terminal_cutover_before_commit_window(&decision_context).await - } else { + if may_publish { commit_terminal_cutover(&decision_context).await + } else { + resolve_terminal_cutover_before_commit_window(&decision_context) } } @@ -356,7 +332,11 @@ async fn snapshot_lease_cutover_state( progress_generation: session.origin_control.progress_generation, media_readiness_generation: session.activity.media_readiness_generation, last_media_progress_at_ms: session.origin_control.last_media_progress_at_ms, - target_duration_ms: session.origin_control.target_duration_snapshot_ms.unwrap_or(manifest_target_duration_ms), + target_duration_ms: session + .origin_control + .target_duration_snapshot_ms + .or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000))) + .unwrap_or(manifest_target_duration_ms), session_target_duration: session.target_duration, observed_latency: session.origin_control.recovery_samples.latency_snapshot(), } @@ -370,19 +350,6 @@ async fn commit_prepared_terminal_decision( preparation: &HlsTerminalTailPreparation, now_ms: u64, ) -> HlsTerminalResolution { - if !preparation.trigger.is_runtime_policy() - && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() - { - return commit_late_custom_response(HlsTerminalCommitContext { - ctx, - session, - proxy_session_id, - lease_id, - preparation, - now_ms, - }) - .await; - } let reason = preparation.trigger.reason(); let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) { Ok(asset) => asset, @@ -408,11 +375,7 @@ pub async fn commit_prepared_runtime_custom_tail( if preparation.trigger.reason() != asset.reason { return HlsTerminalResolution::Reevaluate; } - if !preparation.trigger.is_runtime_policy() - && now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() - { - return commit_late_custom_response(context).await; - } + let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset); let media_asset = asset.asset; let static_incompatibility = if preparation.manifest_snapshot.delivery_mode diff --git a/backend/hls/src/availability/terminal_pending_owner.rs b/backend/hls/src/availability/terminal_pending_owner.rs index 00ad7f6fc..a7f509175 100644 --- a/backend/hls/src/availability/terminal_pending_owner.rs +++ b/backend/hls/src/availability/terminal_pending_owner.rs @@ -66,8 +66,7 @@ pub(super) async fn register_terminal_pending_owner( let Some(owner_key) = terminal_pending_owner_key(context, expected_asset, bundle_key) else { return HlsTerminalResolution::Reevaluate; }; - let latest_safe_commit_at_ms = - context.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let latest_safe_commit_at_ms = owner_key.latest_safe_commit_at_ms; let coordinator = context.ctx.hls_proxy.terminal_pending(); let task_ctx = context.ctx.clone(); let task_session = Arc::clone(context.session); @@ -109,11 +108,11 @@ fn terminal_pending_owner_key( decision_generation: context.preparation.decision_generation, reason: expected_asset.reason, bundle_key, - latest_safe_commit_at_ms: context - .preparation - .cutover_timing - .latest_safe_terminal_commit_at - .as_millis_since_epoch(), + latest_safe_commit_at_ms: if context.preparation.trigger.is_runtime_policy() { + context.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + } else { + context.now_ms.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()) + }, }) } @@ -388,9 +387,6 @@ pub(super) async fn terminal_resolution_with_failed_closed_fallback( if context.preparation.trigger.is_runtime_policy() { return resolution; } - if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed { - return commit_late_custom_response(context).await; - } commit_prepared_terminal_unavailable_after_owner_failure( context.ctx, context.session, @@ -402,44 +398,6 @@ pub(super) async fn terminal_resolution_with_failed_closed_fallback( ) } -pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution { - let prepared = crate::late_custom_response::prepare_late_custom_response( - context.ctx, - context.proxy_session_id, - context.lease_id, - context.preparation, - context.now_ms, - ) - .await; - let (plan, expected_asset) = match prepared { - Ok(prepared) => prepared, - Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => { - return HlsTerminalResolution::Reevaluate - } - Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => { - return commit_prepared_terminal_unavailable_after_owner_failure( - context.ctx, - context.session, - context.proxy_session_id, - context.lease_id, - context.preparation, - context.now_ms, - reason, - ) - } - }; - let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: context.session, - lease_id: context.lease_id, - proxy_session_id: context.proxy_session_id, - preparation: context.preparation, - now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms), - payload: HlsTerminalCommitPayload::CustomResponse(plan), - asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)), - }); - terminal_resolution_for_commit_outcome(outcome, context.now_ms) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum HlsAutonomousTerminalObservation { Committed, diff --git a/backend/hls/src/availability/tests.rs b/backend/hls/src/availability/tests.rs index 3b81087c2..aaa9cc892 100644 --- a/backend/hls/src/availability/tests.rs +++ b/backend/hls/src/availability/tests.rs @@ -225,7 +225,8 @@ mod terminal_fixtures; use terminal_fixtures::{ assert_post_refresh_registration_failure_leaves_no_unowned_live_lease, assert_terminal_pending_registration_failure_commits_unavailable, terminal_base_without_timestamps, - terminal_pending_commit_fixture, terminal_pending_commit_fixture_with_base, terminal_pending_commit_reserve, + terminal_pending_commit_fixture, terminal_pending_commit_fixture_with_base, + terminal_pending_commit_fixture_with_cache, terminal_pending_commit_reserve, }; mod pressure_fixtures; use pressure_fixtures::{ diff --git a/backend/hls/src/availability/tests/late_custom_response.rs b/backend/hls/src/availability/tests/late_custom_response.rs index fc3f001cd..a71483283 100644 --- a/backend/hls/src/availability/tests/late_custom_response.rs +++ b/backend/hls/src/availability/tests/late_custom_response.rs @@ -21,7 +21,12 @@ async fn prepare( fixture.now_ms, ) .await - .unwrap_or_else(|_| panic!("late custom plan")) + .unwrap_or_else(|error| match error { + crate::late_custom_response::HlsLateCustomResponseError::Superseded => panic!("late plan superseded"), + crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason) => { + panic!("late plan unavailable: {reason:?}") + } + }) } fn commit( @@ -42,491 +47,145 @@ fn commit( } #[tokio::test] -async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() { - let fixture = late_fixture("late-without-base").await; - let (plan, asset) = prepare(&fixture).await; - let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media); - assert!( - fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms) - .await - .is_err(), - "preparation cannot authorize uncommitted custom media" - ); - fixture.session.write().await.segments.clear(); - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed); - assert!(!fixture.session.read().await.has_terminal_tail_protections()); - assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n")); - assert!(!plan.manifest_body().contains("/hls/shared/live/")); - let segment = fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms) - .await - .expect("committed late media"); - assert!(!segment.bytes.is_empty()); - fixture.ctx.app_config.custom_stream_response.store(None); - assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted); - let replay = fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1) - .await - .expect("frozen media after config reload"); - assert_eq!(replay.bytes, segment.bytes); - assert!(fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms) - .await - .is_err()); - assert!(fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms) - .await - .is_err()); - assert!(fixture - .ctx - .hls_proxy - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000) - .await - .is_err()); -} - -#[tokio::test] -async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() { - for changed_asset in [false, true] { - let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await; - let (plan, asset) = prepare(&fixture).await; - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - let expected = if changed_asset { - fixture.ctx.app_config.custom_stream_response.store(None); - HlsTerminalCommitOutcome::SupersededGeneration - } else { - fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000); - HlsTerminalCommitOutcome::RecoveryCommitted - }; - assert_eq!(commit(&fixture, plan, asset, now_ms), expected); - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("lease"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - } -} - -#[tokio::test] -async fn late_custom_response_preserves_the_prepared_asset_reason_guard() { - let fixture = late_fixture("late-wrong-asset-reason").await; - let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted) - .expect("configured alternate custom asset"); - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - assert_eq!( - commit_prepared_runtime_custom_tail( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - &fixture.lease_id, - &fixture.preparation, - now_ms, - asset, - ) - .await, - HlsTerminalResolution::Reevaluate - ); +async fn legacy_csv_only_plan_is_rejected_even_after_splice_deadline() { + let fixture = late_fixture("legacy-late-plan-rejected").await; + let (plan, expected_asset) = prepare(&fixture).await; + assert_eq!(commit(&fixture, plan, expected_asset, fixture.now_ms), HlsTerminalCommitOutcome::BundleIncompatible); let lease = fixture .ctx .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) .await .expect("live lease"); assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); } #[tokio::test] -async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() { - let mut fixture = late_fixture("late-target-duration-error").await; - let mut manifest = fixture.preparation.manifest_snapshot.clone(); - manifest.target_duration_ms = 1_000; - manifest.playlist_duration_ms = 1_000; - manifest.last_visible_media_end_ms = 1_000; - Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000; - let guard = fixture - .ctx - .hls_proxy - .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("publication guard"); - assert!(fixture - .ctx - .hls_proxy - .commit_access_lease_manifest_publication( - &fixture.lease_id, - &fixture.proxy_session_id, - guard, - manifest, - fixture.now_ms, - ) - .await - .is_committed()); +async fn exhausted_lease_uses_last_live_segment_after_old_splice_deadline() { + let fixture = late_fixture("late-tail-keeps-live-base").await; + assert!( + fixture.now_ms >= fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() + ); + let previous = fixture.preparation.manifest_snapshot.visible_segments.last().unwrap().clone(); + let context = HlsTerminalCommitContext { + ctx: &fixture.ctx, + session: &fixture.session, + proxy_session_id: &fixture.proxy_session_id, + lease_id: &fixture.lease_id, + preparation: &fixture.preparation, + now_ms: fixture.now_ms, + }; + let bundle = fixture.ready_bundle(); + let outcome = super::super::terminal_cutover::commit_ready_terminal_bundle( + context, + Arc::clone(&fixture.asset), + fixture.expected_asset, + fixture.bundle_key, + bundle, + ) + .await; + assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); let lease = fixture .ctx .hls_proxy .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) .await - .expect("lease with short target duration"); - fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot"); - fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; - let expected = HlsTerminalTailCompatibility::TargetDurationExceeded { - asset_ms: fixture.asset.duration_ms(), - target_ms: 1_000, + .unwrap(); + let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { + panic!("live tail required"); }; - assert!(matches!(prepare_late_custom_response( - &fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms, - ).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected)); - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - assert_eq!( - super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext { + assert_eq!(plan.base_manifest.visible_segments.last(), Some(&previous)); + let manifest = terminal_tail_manifest_body(&plan, &fixture.proxy_session_id, &fixture.lease_id).unwrap(); + let live = manifest.find(previous.uri.as_ref()).expect("last live URI"); + let terminal = manifest.find("/terminal/").expect("CSV tail"); + assert!(live < terminal); + assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); + assert!(!manifest.contains("/cvs/hls/")); + fixture.ctx.hls_proxy.run_garbage_collection_once(fixture.now_ms + 1).await.unwrap(); + let key = SegmentCacheKey::new(fixture.proxy_session_id.clone(), previous.proxy_seq, "ts"); + assert!(fixture.ctx.hls_proxy.segment_cache().metadata(&key).await.unwrap().is_some()); +} + +#[tokio::test] +#[allow( + clippy::too_many_lines, + reason = "Exercise real expiry GC before and after the atomic live-to-terminal protection transfer." +)] +async fn gc_expires_control_bytes_but_keeps_the_last_live_anchor_through_terminal_commit() { + let directory = tempfile::tempdir().unwrap(); + let mut fixture = terminal_pending_commit_fixture_with_cache("gc-live-terminal-anchor", directory.path()).await; + let manager = &fixture.ctx.hls_proxy; + let control = SegmentCacheKey::new(fixture.proxy_session_id.clone(), 39, "ts"); + manager.segment_cache().write_bytes_and_commit(&control, TERMINAL_ASSET_BYTES).await.unwrap(); + { + let mut state = fixture.session.write().await; + let mut entry = state.segments[&40].clone(); + entry.proxy_seq = 39; + entry.cache_key = control.clone(); + entry.origin_key.host_local_sequence = 39; + state.segments.insert(39, entry); + state.proxy_next_seq = Some(41); + let OriginManifestParseOutcome::Normal(origin) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:12,\n100.ts\n#EXTINF:12,\n101.ts\n#EXTINF:12,\n102.ts\n", + "http://origin.example/index.m3u8", + ) else { panic!("normal origin"); }; + state.apply_origin_manifest_for_host(&origin, 1).unwrap(); + state.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; + } + let gc = crate::HlsGarbageCollector::new( + Arc::clone(manager.sessions()), + Arc::clone(manager.segment_cache()), + crate::GarbageCollectionPolicy { + cache_duration_ms: 1, + session_idle_timeout_ms: 1_000_000, + ..crate::GarbageCollectionPolicy::default() + }, + String::new(), + ); + gc.install_access_leases(manager.access_leases()); + let report = gc.run_once(fixture.now_ms + 30_000).await.unwrap(); + assert!( + manager.segment_cache().metadata(&control).await.unwrap().is_none(), + "control must actually be reclaimed: {report:?}" + ); + assert!(manager.segment_cache().metadata(&control).await.unwrap().is_none()); + let live_key = SegmentCacheKey::new(fixture.proxy_session_id.clone(), 40, "ts"); + assert!(manager.segment_cache().metadata(&live_key).await.unwrap().is_some()); + let state = fixture.session.read().await; + let reserve = fixture.preparation.reserve; + let preparation = manager.prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { + lease_id: &fixture.lease_id, + proxy_session_id: &fixture.proxy_session_id, + manifest_snapshot_generation: fixture.preparation.manifest_snapshot_generation, + cursor_generation: fixture.preparation.cursor_generation, + reserve, + cutover_timing: HlsLeaseCutoverTiming::from_reserve(fixture.now_ms, 0, reserve.transition_margin, None), + commit_window: HlsTerminalCommitWindow::CutoverDue, + now_ms: fixture.now_ms, + origin_progress_generation: state.origin_control.progress_generation, + media_readiness_generation: state.activity.media_readiness_generation, + last_media_progress_at_ms: state.origin_control.last_media_progress_at_ms, + }); + drop(state); + fixture.preparation = preparation.await.unwrap(); + let outcome = super::super::terminal_cutover::commit_ready_terminal_bundle( + HlsTerminalCommitContext { ctx: &fixture.ctx, session: &fixture.session, proxy_session_id: &fixture.proxy_session_id, lease_id: &fixture.lease_id, preparation: &fixture.preparation, - now_ms, - }) - .await, - HlsTerminalResolution::Committed - ); - let terminal = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("terminal unavailable lease"); - assert_eq!( - terminal.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { - decision_generation: fixture.preparation.decision_generation, - reason: expected, - } - ); -} - -async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture { - let fixture = late_fixture(name).await; - let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone(); - config.reverse_proxy = - Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto { - hls_cache: Some(shared::model::HlsCacheConfigDto { - session_idle_timeout: shared::model::Secs::new(60), - ..Default::default() - }), - ..Default::default() - })); - fixture.ctx.app_config.config.store(Arc::new(config)); - fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await; - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - assert!(fixture - .ctx - .hls_proxy - .activate_access_lease( - &fixture.lease_id, - &fixture.proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - let (plan, asset) = prepare(&fixture).await; - assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed); - fixture -} - -#[tokio::test] -async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() { - use crate::{ - lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, - HlsMediaActivityCommitOutcome, - }; - let fixture = committed_late_activity_fixture("late-active-media").await; - let manager = &fixture.ctx.hls_proxy; - let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - let before = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms) - .await - .expect("terminal lease"); - let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); - for index in 1_u16..=11 { - let now_ms = started_at_ms + u64::from(index) * 10_000; - let access = manager - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms) - .await - .expect("custom media remains available"); - assert_eq!( - manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await, - HlsMediaActivityCommitOutcome::Committed - ); - if index == 6 { - for key in [ - HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: fixture.lease_id.clone(), - proxy_session_id: fixture.proxy_session_id.clone(), - }, - HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() }, - ] { - manager - .handle_lifecycle_event( - &fixture.ctx.active_users, - &fixture.ctx.active_provider, - HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 }, - now_ms, - ) - .await; - } - } - let lease = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("active terminal lease"); - assert_eq!(lease.state, HlsAccessLeaseState::Activated); - assert_eq!(lease.last_seen_at_ms, now_ms); - assert_eq!(lease.valid_until_ms, now_ms + 60_000); - assert!(lease.active_until_ms.is_some_and(|until| until > now_ms)); - assert_eq!(lease.playback_mode, before.playback_mode); - assert_eq!(lease.playback_cursor, before.playback_cursor); - assert_eq!(lease.admission_generation, before.admission_generation); - assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms); - } - let expired_at_ms = started_at_ms + 170_000; - assert!(manager - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms) - .await - .is_err()); - let expired = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms) - .await - .expect("expired lease"); - assert_eq!(expired.state, HlsAccessLeaseState::Expired); - assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000); -} - -#[tokio::test] -async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() { - use crate::{ - lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey}, - HlsMediaActivityCommitOutcome, - }; - let fixture = committed_late_activity_fixture("late-idle-media").await; - let manager = &fixture.ctx.hls_proxy; - let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - manager - .handle_lifecycle_event( - &fixture.ctx.active_users, - &fixture.ctx.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseActive { - lease_id: fixture.lease_id.clone(), - proxy_session_id: fixture.proxy_session_id.clone(), - }, - due_at_ms: started_at_ms + 30_000, - }, - started_at_ms + 40_000, - ) - .await; - let idle = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) - .await - .expect("idle lease"); - assert_eq!(idle.state, HlsAccessLeaseState::Idle); - let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media); - let mut access = manager - .resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000) - .await - .expect("idle custom media"); - let original_generation = access.terminal_generation; - access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1)); - assert_eq!( - manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - ); - let unchanged = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) - .await - .expect("unchanged idle lease"); - assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms); - assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms); - access.terminal_generation = original_generation; - assert_eq!( - manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await, - HlsMediaActivityCommitOutcome::Committed - ); - let active = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000) - .await - .expect("reactivated lease"); - assert_eq!(active.state, HlsAccessLeaseState::Activated); - assert_eq!(active.valid_until_ms, started_at_ms + 100_000); - assert_eq!( - manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - ); - let expired = manager - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000) - .await - .expect("expired lease"); - assert_eq!(expired.state, HlsAccessLeaseState::Expired); - assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms); - assert_eq!(expired.valid_until_ms, active.valid_until_ms); -} - -#[tokio::test] -async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() { - let mut fixture = late_fixture("late-contended-commit").await; - let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms(); - fixture.preparation.cutover_timing = - HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None); - let (plan, asset) = prepare(&fixture).await; - let guard = fixture.session.read().await; - assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); - assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. })); - drop(guard); - let committed = tokio::time::timeout(Duration::from_secs(3), async { - loop { - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - fixture.ctx.hls_proxy.terminal_commit_now_ms(), - ) - .await - .expect("lease"); - if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode { - break committed; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("autonomous late custom commit"); - assert_eq!(committed, plan); - assert_eq!( - commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()), - HlsTerminalCommitOutcome::AlreadyCommitted - ); -} - -#[tokio::test] -async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() { - let mut fixture = late_fixture("late-highwater").await; - for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] { - let guard = fixture - .ctx - .hls_proxy - .prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("publication guard"); - let mut manifest = fixture.preparation.manifest_snapshot.clone(); - manifest.first_proxy_seq = last_seq; - manifest.last_proxy_seq = last_seq; - manifest.discontinuity_sequence = discontinuity_sequence; - let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment"); - segment.proxy_seq = last_seq; - segment.discontinuity_before = discontinuity_before; - assert!(fixture - .ctx - .hls_proxy - .commit_access_lease_manifest_publication( - &fixture.lease_id, - &fixture.proxy_session_id, - guard, - manifest, - fixture.now_ms - ) - .await - .is_committed()); - } - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("lease"); - fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot"); - fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation; - let (plan, _) = prepare(&fixture).await; - assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n")); - assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n")); -} - -#[tokio::test] -async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() { - use crate::terminal_commit::{ - terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey, - HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision, - }; - let fixture = late_fixture("late-bounded-cas").await; - let (plan, asset) = prepare(&fixture).await; - let coordinator = HlsTerminalCommitRetryCoordinator::default(); - let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - let command = |decision| { - let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding"); - HlsTerminalCommitCommand { - key: HlsTerminalCommitOwnerKey { - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - lease_issued_at_ms: fixture.preparation.lease_issued_at_ms, - decision_generation: fixture.preparation.decision_generation, - }, - session: Arc::clone(&fixture.session), - session_incarnation: fixture - .ctx - .hls_proxy - .sessions() - .session_incarnation(&fixture.session) - .expect("session incarnation"), - preparation: fixture.preparation.clone(), - decision, - media_guard: None, - asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)), - cancellation_epoch, - submission_token, - } - }; - assert!(matches!( - coordinator.submit( - command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), - now_ms - ), - HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) - )); - assert!(matches!( - coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms), - HlsTerminalCommitSubmissionDecision::Attempt { .. } - )); - assert!(matches!( - coordinator.submit( - command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)), - now_ms + 1 - ), - HlsTerminalCommitSubmissionDecision::PendingExisting { .. } - )); - assert!(matches!( - coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100), - HlsTerminalCommitSubmissionDecision::PendingExisting { .. } - )); - assert!(matches!( - coordinator.submit( - command(HlsTerminalLeaseDecision::CustomResponse(plan)), - now_ms + terminal_commit_retry_schedule_budget_ms() - ), - HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed) - )); + now_ms: fixture.now_ms, + }, + Arc::clone(&fixture.asset), + fixture.expected_asset, + fixture.bundle_key, + fixture.ready_bundle(), + ) + .await; + assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); + assert!(!fixture.session.read().await.live_window_demands.contains_key(&fixture.lease_id)); + gc.run_once(fixture.now_ms + 31_000).await.unwrap(); + let metadata = manager.segment_cache().metadata(&live_key).await.unwrap().unwrap(); + assert_eq!(tokio::fs::read(metadata.path).await.unwrap(), TERMINAL_ASSET_BYTES); } diff --git a/backend/hls/src/availability/tests/owner_fixtures.rs b/backend/hls/src/availability/tests/owner_fixtures.rs index 9a5860cd5..ba5fe6050 100644 --- a/backend/hls/src/availability/tests/owner_fixtures.rs +++ b/backend/hls/src/availability/tests/owner_fixtures.rs @@ -338,7 +338,7 @@ pub(super) async fn post_refresh_terminal_fixture_with_bundle_state( &session, &proxy_session_id, &lease_id, - if terminal_asset { 22_000 } else { 7_000 }, + if terminal_asset { 36_000 } else { 16_000 }, now_ms, ) .await; diff --git a/backend/hls/src/availability/tests/pressure_fixtures.rs b/backend/hls/src/availability/tests/pressure_fixtures.rs index 243712dd8..a8bcaf2e5 100644 --- a/backend/hls/src/availability/tests/pressure_fixtures.rs +++ b/backend/hls/src/availability/tests/pressure_fixtures.rs @@ -69,7 +69,6 @@ pub(super) fn evaluated_pressure( cursor: &HlsLeasePlaybackCursor::default(), ready_timeline: &ready_timeline, now_ms: 100, - playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget, origin_path_degraded: true, recovery_committed: false, diff --git a/backend/hls/src/availability/tests/recovery.rs b/backend/hls/src/availability/tests/recovery.rs index cebe93d8f..4353de621 100644 --- a/backend/hls/src/availability/tests/recovery.rs +++ b/backend/hls/src/availability/tests/recovery.rs @@ -327,7 +327,7 @@ fn hls_session_recovery_pressure_new_urgent_lease_controls_atomic_commit() { .expect("urgent pressure"); assert!(after.decision.evaluate_lease_cutovers); - assert_eq!(after.timing_seed.target_duration_ms, 10_000); + assert_eq!(after.timing_seed.target_duration_ms, 8_000); } #[test] @@ -345,7 +345,7 @@ fn hls_session_recovery_pressure_expired_controller_is_excluded_atomically() { atomic_pressure_policy(), ) .expect("urgent pressure"); - assert_eq!(urgent.timing_seed.target_duration_ms, 10_000); + assert_eq!(urgent.timing_seed.target_duration_ms, 8_000); let after_expiry = evaluate_and_commit_session_recovery_pressure( &mut leases, @@ -403,7 +403,7 @@ async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeas } }) .await - .expect("terminal decision before reserve is exhausted") + .expect("terminal decision at reserve exhaustion") } #[tokio::test] @@ -414,7 +414,7 @@ async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh &fixture.session, &fixture.proxy_session_id, &fixture.lease_id, - 18_000, + 33_000, fixture.now_ms, ) .await; @@ -440,11 +440,11 @@ async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh assert_eq!(result, HlsTerminalResolution::LiveAllowed); assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); - // R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late. + // R=3,000 ms: the independent owner must wait for exhaustion, without another poll. // Only read the resulting state; do not issue another terminal/manifest request. let mode = wait_for_terminal_mode(&fixture).await; assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); - assert!(current_time_millis() < fixture.now_ms + 5_100); + assert!(fixture.ctx.hls_proxy.now_ms() >= fixture.now_ms + 3_000); assert!(fixture.session.read().await.origin_refresh.in_flight); } @@ -457,7 +457,7 @@ async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() { &fixture.session, &fixture.proxy_session_id, &fixture.lease_id, - 18_000, + 33_000, fixture.now_ms, ) .await; @@ -505,7 +505,7 @@ async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() { let mode = wait_for_terminal_mode(&fixture).await; assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}"); - assert!(current_time_millis() < fixture.now_ms + 5_100); + assert!(fixture.ctx.hls_proxy.now_ms() >= fixture.now_ms + 3_000); assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry"); assert!(fixture.session.read().await.origin_refresh.in_flight); assert!(tokio::time::timeout(Duration::from_secs(8), refresh) @@ -523,7 +523,7 @@ async fn recovered_origin_cancels_the_retained_reserve_cutover() { &fixture.session, &fixture.proxy_session_id, &fixture.lease_id, - 18_000, + 33_000, fixture.now_ms, ) .await; @@ -558,14 +558,14 @@ async fn recovered_origin_cancels_the_retained_reserve_cutover() { } #[tokio::test] -async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() { +async fn expired_splice_deadline_keeps_last_live_segment_when_reserve_is_exhausted() { let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await; advance_post_refresh_fixture_playback( &fixture.ctx, &fixture.session, &fixture.proxy_session_id, &fixture.lease_id, - 26_000, + 36_000, fixture.now_ms, ) .await; @@ -592,7 +592,7 @@ async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leavin .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) .await .expect("terminal lease"); - assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); assert_eq!( commit_terminal_tail_if_lease_reserve_requires_cutover( &fixture.ctx, @@ -604,12 +604,21 @@ async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leavin .await, HlsTerminalResolution::Committed ); - assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert!(fixture.session.read().await.has_terminal_tail_protections()); } #[tokio::test(start_paused = true)] async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() { let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await; + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 0, + fixture.now_ms, + ) + .await; fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms); let lease = fixture .ctx @@ -633,14 +642,14 @@ async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_lo } #[tokio::test] -async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() { +async fn publication_late_origin_keeps_live_base_after_the_splice_deadline() { let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await; advance_post_refresh_fixture_playback( &fixture.ctx, &fixture.session, &fixture.proxy_session_id, &fixture.lease_id, - 26_000, + 36_000, fixture.now_ms, ) .await; @@ -668,5 +677,5 @@ async fn publication_late_origin_can_finish_custom_response_after_the_splice_dea .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()) .await .expect("terminal lease"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_))); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); } diff --git a/backend/hls/src/availability/tests/runtime_fixtures.rs b/backend/hls/src/availability/tests/runtime_fixtures.rs index a85538356..fc572b38e 100644 --- a/backend/hls/src/availability/tests/runtime_fixtures.rs +++ b/backend/hls/src/availability/tests/runtime_fixtures.rs @@ -256,6 +256,15 @@ pub(super) async fn add_active_post_refresh_lease( ) .await .is_activated()); + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease_id, + 36_000, + fixture.now_ms, + ) + .await; } pub(super) async fn advance_post_refresh_fixture_playback( @@ -285,6 +294,19 @@ pub(super) async fn advance_post_refresh_fixture_playback( ) .await .expect("fixture segment request starts"); + assert_eq!( + ctx.hls_proxy + .mark_delivered_media_for_lease( + session, + lease_id, + proxy_session_id, + identity, + Some(token), + playback_at_ms, + ) + .await, + crate::manager::HlsMediaActivityCommitOutcome::Committed + ); assert_eq!( ctx.hls_proxy .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( diff --git a/backend/hls/src/availability/tests/terminal.rs b/backend/hls/src/availability/tests/terminal.rs index 98642ace8..61a5ab4f3 100644 --- a/backend/hls/src/availability/tests/terminal.rs +++ b/backend/hls/src/availability/tests/terminal.rs @@ -39,7 +39,7 @@ fn autonomous_terminal_live_allowed_means_no_cutover_is_required() { } #[test] -fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() { +fn live_reserve_wake_is_at_exhaustion_without_an_immediate_margin_loop() { let now_ms = 1_000; let mut reserve = terminal_pending_commit_reserve(); reserve.guaranteed_reserve_ms = reserve.guaranteed_reserve_ms.saturating_add(5_000); @@ -49,7 +49,7 @@ fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() { let deadline = live_reserve_deadline(now_ms, reserve, cutover_timing).expect("future acquisition wake"); assert_eq!(deadline.next_reevaluation_at_ms, now_ms.saturating_add(5_000)); - assert!(deadline.next_reevaluation_at_ms < deadline.latest_safe_terminal_commit_at_ms); + assert_eq!(deadline.next_reevaluation_at_ms, deadline.latest_safe_terminal_commit_at_ms); } #[tokio::test] diff --git a/backend/hls/src/availability/tests/terminal_fixtures.rs b/backend/hls/src/availability/tests/terminal_fixtures.rs index 6b5fc6dca..6a81a48b6 100644 --- a/backend/hls/src/availability/tests/terminal_fixtures.rs +++ b/backend/hls/src/availability/tests/terminal_fixtures.rs @@ -19,10 +19,8 @@ impl HlsTerminalPendingCommitFixture { reason: self.expected_asset.reason, bundle_key: self.bundle_key, latest_safe_commit_at_ms: self - .preparation - .cutover_timing - .latest_safe_terminal_commit_at - .as_millis_since_epoch(), + .now_ms + .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()), } } @@ -69,19 +67,18 @@ impl HlsTerminalPendingCommitFixture { pub(super) fn terminal_pending_commit_reserve() -> HlsLeaseReserveSnapshot { let transition_margin = HlsTransitionMarginMs::from_millis(12_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + let guaranteed_reserve_ms = 0; HlsLeaseReserveSnapshot { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: guaranteed_reserve_ms, conservative_playback_position_ms: 0, + playback_position_known: true, guaranteed_reserve_ms, initial_hidden_ready_duration_ms: 0, transition_margin, key_readiness_valid_until_ms: None, recovery_required: true, - cutover_required: false, + cutover_required: true, } } @@ -137,6 +134,7 @@ pub(super) async fn install_terminal_pending_ready_base( .await .expect("READY terminal-base bytes commit"); let mut session = session.write().await; + session.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure; let origin_epoch = session.origin_control.origin_epoch; session.segments.insert( 40, @@ -214,7 +212,25 @@ pub(super) async fn terminal_pending_commit_fixture_with_base( name: &str, base_segment_bytes: &[u8], ) -> HlsTerminalPendingCommitFixture { - let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); + let ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); + terminal_pending_commit_fixture_with_ctx(name, base_segment_bytes, ctx).await +} + +pub(super) async fn terminal_pending_commit_fixture_with_cache( + name: &str, + path: &std::path::Path, +) -> HlsTerminalPendingCommitFixture { + let mut ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); + ctx.hls_proxy = Arc::new(crate::HlsProxyManager::with_cache_settings(path, 300)); + ctx.attach_lease_user_accounting(); + terminal_pending_commit_fixture_with_ctx(name, TERMINAL_ASSET_BYTES, ctx).await +} + +async fn terminal_pending_commit_fixture_with_ctx( + name: &str, + base_segment_bytes: &[u8], + hls_ctx: HlsCtx, +) -> HlsTerminalPendingCommitFixture { let ctx = &hls_ctx; let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); let asset = snapshot_terminal_media_asset(&terminal_buffer).expect("valid terminal asset fixture"); @@ -333,6 +349,10 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail )); } +#[allow( + clippy::too_many_lines, + reason = "Full reserve exhaustion and owner-failure lifecycle share this terminal fixture." +)] pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( failure: HlsAvailabilityReevaluationRegistration, name: &str, @@ -385,6 +405,65 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l ) .await .is_activated()); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .unwrap(); + let identity = lease.media_identity().unwrap(); + let started_at_ms = fixture.now_ms - 100_000; + let token = fixture + .ctx + .hls_proxy + .record_access_lease_segment_request_started_if_identity_matches( + &fixture.lease_id, + &fixture.proxy_session_id, + identity, + 40, + started_at_ms, + ) + .await + .unwrap(); + assert_eq!( + fixture + .ctx + .hls_proxy + .mark_delivered_media_for_lease( + &fixture.session, + &fixture.lease_id, + &fixture.proxy_session_id, + identity, + Some(token), + started_at_ms + ) + .await, + crate::manager::HlsMediaActivityCommitOutcome::Committed + ); + assert_eq!( + fixture + .ctx + .hls_proxy + .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + &fixture.session, + &fixture.lease_id, + &fixture.proxy_session_id, + identity, + token, + started_at_ms + ) + .await, + crate::manager::HlsMediaActivityCommitOutcome::Committed + ); + assert!(fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .unwrap() + .playback_cursor + .first_segment_started_at_ms + .is_some()); let outcome = commit_post_refresh_terminal_fallback( fixture.ctx.clone(), Arc::clone(&fixture.session), @@ -397,7 +476,32 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l ) .await; - assert_eq!(outcome, HlsPostRefreshFallbackOutcome::TerminalCommitted); + assert!(matches!( + outcome, + HlsPostRefreshFallbackOutcome::TerminalCommitted | HlsPostRefreshFallbackOutcome::PendingOwnerRegistered + )); + if outcome == HlsPostRefreshFallbackOutcome::PendingOwnerRegistered { + tokio::time::timeout(Duration::from_secs(4), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + fixture.ctx.hls_proxy.now_ms(), + ) + .await + .unwrap(); + if lease.playback_mode != HlsLeasePlaybackMode::Live { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("owned terminal decision finishes"); + } let lease = fixture .ctx .hls_proxy @@ -406,6 +510,6 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l .expect("fallback lease remains stored"); assert!(matches!( lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. } + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } )); } diff --git a/backend/hls/src/critical_handoff.rs b/backend/hls/src/critical_handoff.rs index 3112948f1..de26ffb71 100644 --- a/backend/hls/src/critical_handoff.rs +++ b/backend/hls/src/critical_handoff.rs @@ -9,7 +9,7 @@ use super::{ HlsTerminalTailCompatibilityInput, }, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsCriticalHandoffStateAccess, HlsLeaseReserveInput, - HlsMediaLeaseIdentity, HlsSession, HlsTsTrackSignature, SegmentCacheStatus, HLS_PLAYBACK_RATE_GUARD_MILLI, + HlsMediaLeaseIdentity, HlsSession, HlsTsTrackSignature, SegmentCacheStatus, }; use log::{debug, warn}; use std::{future::Future, sync::Arc}; @@ -88,10 +88,6 @@ struct HlsCriticalHandoffCandidate { media_exhaustion_deadline_ms: u64, } -fn guarded_playback_wall_time_ms(media_time_ms: u64) -> u64 { - media_time_ms.saturating_mul(1_000) / u64::from(HLS_PLAYBACK_RATE_GUARD_MILLI.max(1)) -} - pub(super) fn select_critical_handoff_lease( session: &HlsSession, leases: &[HlsAccessLease], @@ -119,7 +115,6 @@ pub(super) fn select_critical_handoff_lease( cursor: &lease.playback_cursor, ready_timeline: &ready_timeline, now_ms, - playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0), origin_path_degraded: true, recovery_committed: false, @@ -131,10 +126,9 @@ pub(super) fn select_critical_handoff_lease( entry.proxy_file_ext.eq_ignore_ascii_case("ts") && matches!(&entry.status, SegmentCacheStatus::Ready { .. }) })?; - let media_exhaustion_deadline_ms = - now_ms.saturating_add(guarded_playback_wall_time_ms(reserve.guaranteed_reserve_ms)); - let safe_cutover_deadline_ms = media_exhaustion_deadline_ms - .saturating_sub(guarded_playback_wall_time_ms(reserve.transition_margin.as_millis())); + let media_exhaustion_deadline_ms = now_ms.saturating_add(reserve.guaranteed_reserve_ms); + let safe_cutover_deadline_ms = + media_exhaustion_deadline_ms.saturating_sub(reserve.transition_margin.as_millis()); Some(HlsCriticalHandoffCandidate { lease: lease.clone(), snapshot: HlsCriticalHandoffSnapshot { diff --git a/backend/hls/src/cutover.rs b/backend/hls/src/cutover.rs index 6ec787ecc..a39ab9f92 100644 --- a/backend/hls/src/cutover.rs +++ b/backend/hls/src/cutover.rs @@ -37,16 +37,15 @@ pub enum HlsTerminalCutoverDecision { /// Decides terminal publication from one immutable lease/session snapshot. /// /// A cutover is authorized only by real media reserve and a current snapshot. -/// Acceptance lifecycle states never stand in for media progress: once the -/// the bounded terminal-acquisition window opens, missing, active, exhausted, -/// and committed episodes all continue to terminal-capability evaluation. A structurally -/// superseded snapshot is retried so reserve can be evaluated again. -/// Publication lateness, request counts, -/// recovery ETA, hard operation timeouts, and generic refresh activity are -/// deliberately absent. Pre-cutover scheduling owns all waits; once the -/// acquisition window opens, neither recovery nor terminal-media preparation -/// may postpone the terminal decision beyond the unchanged safe deadline. +/// Publication requires known reserve exhaustion. Acquisition windows and the +/// transition margin may prepare recovery/media earlier, but never authorize CSV. +/// Missing, active, exhausted and committed acceptance episodes can all reach +/// capability evaluation at exhaustion; only a structurally superseded snapshot +/// requires fresh reserve evidence. An in-flight retry cannot delay this decision. pub fn evaluate_terminal_cutover(input: &HlsTerminalCutoverInput) -> HlsTerminalCutoverDecision { + if !input.reserve.is_exhausted() { + return HlsTerminalCutoverDecision::NotRequired; + } match input.commit_window { HlsTerminalCommitWindow::NotDue => return HlsTerminalCutoverDecision::NotRequired, HlsTerminalCommitWindow::AcquisitionOpen | HlsTerminalCommitWindow::CutoverDue => {} @@ -116,6 +115,7 @@ mod tests { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: guaranteed_reserve_ms, conservative_playback_position_ms: 0, + playback_position_known: true, guaranteed_reserve_ms, initial_hidden_ready_duration_ms: 0, transition_margin, @@ -127,7 +127,7 @@ mod tests { fn input(acceptance: HlsManifestAcceptanceEpisodeStatus) -> HlsTerminalCutoverInput { HlsTerminalCutoverInput { - reserve: degraded_reserve(10_000), + reserve: degraded_reserve(0), commit_window: HlsTerminalCommitWindow::CutoverDue, acceptance, required_terminal_media_key: Some(TERMINAL_KEY), @@ -143,6 +143,22 @@ mod tests { } } + #[test] + fn positive_or_unknown_reserve_never_authorizes_csv_even_during_retries() { + let acquisition = crate::recovery_timing::HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis(); + for reserve_ms in [10_000 + acquisition, 10_000, 1] { + for window in [HlsTerminalCommitWindow::AcquisitionOpen, HlsTerminalCommitWindow::CutoverDue] { + let mut candidate = input(HlsManifestAcceptanceEpisodeStatus::InFlight { generation: GENERATION }); + candidate.reserve = degraded_reserve(reserve_ms); + candidate.commit_window = window; + assert_eq!(evaluate_terminal_cutover(&candidate), HlsTerminalCutoverDecision::NotRequired); + } + } + let mut unknown = input(exhausted()); + unknown.reserve.playback_position_known = false; + assert_eq!(evaluate_terminal_cutover(&unknown), HlsTerminalCutoverDecision::NotRequired); + } + #[test] fn hls_cutover_policy_full_burst_exhaustion_requires_terminal_capability() { let mut undecided = input(exhausted()); @@ -250,13 +266,13 @@ mod tests { } #[test] - fn hls_cutover_policy_acquisition_window_can_commit_before_the_unchanged_margin() { + fn hls_cutover_policy_acquisition_window_cannot_publish_with_positive_reserve() { let mut acquisition = input(exhausted()); acquisition.reserve = degraded_reserve(10_001); acquisition.commit_window = HlsTerminalCommitWindow::AcquisitionOpen; assert!(!acquisition.reserve.cutover_required); - assert_eq!(evaluate_terminal_cutover(&acquisition), HlsTerminalCutoverDecision::CommitTerminalTail); + assert_eq!(evaluate_terminal_cutover(&acquisition), HlsTerminalCutoverDecision::NotRequired); } #[test] diff --git a/backend/hls/src/gc/policy.rs b/backend/hls/src/gc/policy.rs index 2b7a87d9a..ffff4ffd6 100644 --- a/backend/hls/src/gc/policy.rs +++ b/backend/hls/src/gc/policy.rs @@ -63,6 +63,12 @@ impl ProtectedSet { if let Some(rendered) = &session.last_rendered_manifest { protected.key_resource_ids.extend(extract_transient_resource_ids(&rendered.body)); } + // The last committed live segment is the mandatory terminal splice + // anchor. Capacity recovery may release consumed history, never this + // anchor while its lease remains valid. + protected + .segment_proxy_seqs + .extend(session.live_window_demands.values().filter_map(|demand| demand.published.last().copied())); for (_, protection) in session.terminal_tail_protections() { protected.segment_proxy_seqs.extend(protection.base_proxy_seqs.iter().copied()); } diff --git a/backend/hls/src/gc/tests/publication.rs b/backend/hls/src/gc/tests/publication.rs index f71db4462..f7ee23eff 100644 --- a/backend/hls/src/gc/tests/publication.rs +++ b/backend/hls/src/gc/tests/publication.rs @@ -223,6 +223,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { for seq in 0..8 { let token = cursor.record_request_started(seq, 1_000 + seq * 10_000); let token = cursor.bind_request_timing(token, &session.segments[&seq]); + cursor.record_transfer_started(token, 1_000 + seq * 10_000); let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000); } let manifest = HlsLeaseManifestSnapshot { @@ -261,7 +262,6 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { cursor: &cursor, ready_timeline: &timeline, now_ms: 71_000, - playback_rate_guard_milli: 1_050, recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000), origin_path_degraded: true, recovery_committed: false, @@ -269,7 +269,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { (timeline.units.len(), reserve) }; let (before_units, before) = evaluate(&session); - assert_eq!(before.guaranteed_reserve_ms, 34_500); + assert_eq!(before.guaranteed_reserve_ms, 38_000); assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0)); // Run the actual metadata-removal function used by garbage collection. assert!(remove_segment_entry(&mut session, 0).is_some()); @@ -282,7 +282,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { assert_eq!(after, before); let timing = HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None); - assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500); + assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 97_000); // Removing the entire historical prefix must keep exactly the same media clock. for seq in 1..4 { assert!(remove_segment_entry(&mut session, seq).is_some()); @@ -290,7 +290,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() { assert_eq!(evaluate(&session).1, before); // A real hole in the current READY span must still shorten the reserve. assert!(remove_segment_entry(&mut session, 7).is_some()); - assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500); + assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 26_000); } fn install_fast_start(session: &mut HlsSession) { @@ -510,6 +510,10 @@ async fn obsolete_metadata_is_removed_only_after_publication_and_reader_protecti published: vec![1], }, ); + assert!( + ProtectedSet::from_session_for_capacity(&state, Some(u64::MAX)).segment_proxy_seqs.contains(&1), + "capacity recovery must retain the lease's last live anchor" + ); state.install_terminal_tail_protection( lease_id.clone(), HlsTerminalTailProtection { diff --git a/backend/hls/src/late_custom_response.rs b/backend/hls/src/late_custom_response.rs index 2233a80e9..c59649a90 100644 --- a/backend/hls/src/late_custom_response.rs +++ b/backend/hls/src/late_custom_response.rs @@ -1,5 +1,8 @@ -//! Finite custom media published after the live splice deadline has elapsed. +//! Legacy standalone lease-plan shape. Published live leases reject this plan; +//! its constructors exist only for regression tests. Unpublished standalone +//! responses remain owned by `runtime_custom_tail`. +#[cfg(test)] use crate::{ hls_ctx::HlsCtx, lease::{HlsAccessLeaseId, HlsTerminalTailPreparation}, @@ -12,7 +15,17 @@ use crate::{ terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration}, HlsAccessLeaseState, ProxySessionId, }; -use std::{fmt, fmt::Write, sync::Arc}; +#[cfg(not(test))] +use crate::{ + lease::HlsAccessLeaseId, + runtime_custom_tail::{standalone_custom_asset_fingerprint, HlsStandaloneCustomPlan, HlsStandaloneCustomPlanKey}, + terminal_tail::HlsTerminalTailGeneration, + ProxySessionId, +}; +use std::fmt; +#[cfg(test)] +use std::{fmt::Write, sync::Arc}; +#[cfg(test)] use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs}; /// Immutable, lease-bound custom playlist that does not consume a live splice base. @@ -69,11 +82,13 @@ impl PartialEq for HlsLateCustomResponsePlan { impl Eq for HlsLateCustomResponsePlan {} +#[cfg(test)] pub(crate) enum HlsLateCustomResponseError { Superseded, Unavailable(HlsTerminalTailCompatibility), } +#[cfg(test)] pub(crate) async fn prepare_late_custom_response( ctx: &HlsCtx, proxy_session_id: &ProxySessionId, @@ -144,6 +159,7 @@ pub(crate) async fn prepare_late_custom_response( } // Keeping the layout separate makes the publication boundary testable without a live origin or cache. +#[cfg(test)] fn render_late_manifest( first_proxy_seq: u64, discontinuity_sequence: u64, diff --git a/backend/hls/src/lease/lifecycle.rs b/backend/hls/src/lease/lifecycle.rs index d04d5cf29..a8afe6bd4 100644 --- a/backend/hls/src/lease/lifecycle.rs +++ b/backend/hls/src/lease/lifecycle.rs @@ -88,6 +88,10 @@ impl HlsAccessLease { pub(super) fn end_playback(&mut self) { self.runtime_policy_revocation = None; + self.revision_bindings = None; + if let Some(snapshot) = self.last_manifest_snapshot.as_mut() { + snapshot.startup_revisions = None; + } let release_generation = match std::mem::replace(&mut self.playback_mode, HlsLeasePlaybackMode::Ended) { HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.generation), HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => { @@ -341,6 +345,31 @@ impl HlsAccessLeaseStore { .collect() } + /// A valid idle lease can still be consuming its previously available + /// media. The terminal observer must survive a pause in playlist polls. + pub(crate) fn terminal_playback_snapshots_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Vec { + if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { + return Vec::new(); + } + self.by_lease_id + .values() + .filter(|lease| { + lease.proxy_session_id == *proxy_session_id + && ((lease.state == HlsAccessLeaseState::Activated + && lease.active_until_ms.is_some_and(|until| until > now_ms)) + || (matches!(lease.state, HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) + && lease.playback_cursor.first_segment_started_at_ms.is_some())) + && lease.playback_mode == HlsLeasePlaybackMode::Live + && lease.last_manifest_snapshot.is_some() + }) + .cloned() + .collect() + } + pub fn record_segment_request_started_if_identity_matches( &mut self, lease_id: &HlsAccessLeaseId, @@ -390,6 +419,31 @@ impl HlsAccessLeaseStore { Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment)) } + pub fn record_segment_transfer_started_if_identity_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + token: HlsPlaybackRequestToken, + started_at_ms: u64, + ) -> Option { + if !expected.is_live() || !self.media_identity_is_current(lease_id, proxy_session_id, expected, started_at_ms) { + return None; + } + let mut cursor = self.by_lease_id.get(lease_id)?.playback_cursor.clone(); + let previous_generation = cursor.cursor_generation; + if !cursor.record_transfer_started(token, started_at_ms) { + return None; + } + let changed = cursor.cursor_generation != previous_generation; + if changed { + self.availability_evidence_can_advance().ok()?; + self.by_lease_id.get_mut(lease_id)?.playback_cursor = cursor; + self.advance_availability_evidence(proxy_session_id).ok()?; + } + Some(changed) + } + pub fn record_segment_request_completed_if_identity_matches( &mut self, lease_id: &HlsAccessLeaseId, diff --git a/backend/hls/src/lease/terminal.rs b/backend/hls/src/lease/terminal.rs index 06d43e642..8e0f0d5bd 100644 --- a/backend/hls/src/lease/terminal.rs +++ b/backend/hls/src/lease/terminal.rs @@ -50,6 +50,9 @@ impl HlsAccessLease { preparation: &HlsTerminalTailPreparation, now_ms: u64, ) -> Result<(), HlsTerminalCommitOutcome> { + if !preparation.trigger.is_runtime_policy() && !preparation.reserve.is_exhausted() { + return Err(HlsTerminalCommitOutcome::CutoverNoLongerRequired); + } match preparation.runtime_policy_revocation.as_ref() { Some(revocation) => { self.runtime_policy_tail_commit_preconditions_match(proxy_session_id, preparation, revocation) @@ -256,6 +259,19 @@ impl HlsAccessLeaseStore { ) -> HlsTerminalCommitOutcome { if !plan.matches_route(proxy_session_id, lease_id) || plan.generation.0 != preparation.decision_generation + || (!preparation.trigger.is_runtime_policy() + && (plan.base_manifest.last_proxy_seq != preparation.manifest_snapshot.last_proxy_seq + || plan + .base_manifest + .visible_segments + .last() + .zip(preparation.manifest_snapshot.visible_segments.last()) + .is_none_or(|(actual, expected)| { + actual.proxy_seq != expected.proxy_seq + || actual.uri != expected.uri + || actual.duration_ms != expected.duration_ms + || actual.encryption != expected.encryption + }))) || plan.base_manifest.snapshot_generation != preparation.manifest_snapshot_generation { return HlsTerminalCommitOutcome::SupersededGeneration; @@ -298,40 +314,15 @@ impl HlsAccessLeaseStore { proxy_session_id: &ProxySessionId, preparation: &HlsTerminalTailPreparation, now_ms: u64, - plan: std::sync::Arc, + plan: &crate::late_custom_response::HlsLateCustomResponsePlan, ) -> HlsTerminalCommitOutcome { - if !plan.matches_route(proxy_session_id, lease_id) - || plan.generation.0 != preparation.decision_generation - || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation - || preparation.trigger.is_runtime_policy() - { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + let Some(lease) = self.response_snapshot(lease_id, proxy_session_id, now_ms) else { return HlsTerminalCommitOutcome::LeaseNoLongerEligible; }; - if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { - return outcome; - } - if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) - { - return outcome; - } - if self.availability_evidence_can_advance().is_err() { + if lease.issued_at_ms != preparation.lease_issued_at_ms || !plan.matches_route(proxy_session_id, lease_id) { return HlsTerminalCommitOutcome::SupersededGeneration; } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsTerminalCommitOutcome::LeaseNoLongerEligible; - }; - lease.admission_generation = preparation.decision_generation; - lease.playback_mode = HlsLeasePlaybackMode::TerminalCustomResponse(plan); - if self.advance_availability_evidence(proxy_session_id).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - HlsTerminalCommitOutcome::Committed + HlsTerminalCommitOutcome::BundleIncompatible } pub(crate) fn late_custom_segment_access( diff --git a/backend/hls/src/lease/tests/terminal.rs b/backend/hls/src/lease/tests/terminal.rs index 996a43fca..306eb2acf 100644 --- a/backend/hls/src/lease/tests/terminal.rs +++ b/backend/hls/src/lease/tests/terminal.rs @@ -1,12 +1,12 @@ use super::{ - build_terminal_tail_plan, lease, manifest_snapshot, publish_manifest_snapshot, snapshot_terminal_media_asset, - timing, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeaseState, - HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeaseCutoverTiming, HlsLeaseManifestPublicationOutcome, - HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, - HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestAcceptanceGeneration, - HlsPlaybackCompletionOutcome, HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, - HlsRuntimePolicyRevocationOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, - HlsTerminalBaseSegmentAvailability, HlsTerminalCommitOutcome, HlsTerminalCommitWindow, + build_terminal_tail_plan, lease, manifest_snapshot, publish_manifest_snapshot as publish_unrouted_manifest, + snapshot_terminal_media_asset, timing, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, + HlsAccessLeaseState, HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeaseCutoverTiming, + HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSegment, + HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, + HlsManifestAcceptanceGeneration, HlsPlaybackCompletionOutcome, HlsRuntimeCustomTailAssetIdentity, + HlsRuntimeCustomTailReason, HlsRuntimePolicyRevocationOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, + HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, HlsTerminalCommitOutcome, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, HlsTerminalMediaRequirementOrigin, HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailPreparationInput, HlsTransitionMarginMs, @@ -15,6 +15,19 @@ use crate::ProxySessionId; use std::sync::Arc; use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; +fn publish_manifest_snapshot( + store: &mut HlsAccessLeaseStore, + lease_id: &HlsAccessLeaseId, + session: &ProxySessionId, + mut snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsLeaseManifestPublicationOutcome { + for segment in Arc::make_mut(&mut snapshot.visible_segments) { + segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", session.0, lease_id.0, segment.proxy_seq).into(); + } + publish_unrouted_manifest(store, lease_id, session, snapshot, now_ms) +} + pub(in crate::lease::tests) const TERMINAL_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); @@ -23,6 +36,7 @@ pub(in crate::lease::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: 12_000, conservative_playback_position_ms: 12_000, + playback_position_known: true, guaranteed_reserve_ms: 0, initial_hidden_ready_duration_ms: 0, transition_margin: HlsTransitionMarginMs::from_millis(12_000), @@ -36,6 +50,40 @@ pub(in crate::lease::tests) fn cutover_timing() -> HlsLeaseCutoverTiming { HlsLeaseCutoverTiming::from_reserve(2_000, 0, HlsTransitionMarginMs::from_millis(12_000), None) } +#[test] +fn terminal_monitor_keeps_started_idle_leases_until_validity_expires() { + let session = ProxySessionId("idle-terminal-monitor".into()); + let lease_id = HlsAccessLeaseId("idle-terminal-monitor".into()); + let mut leases = HlsAccessLeaseStore::default(); + leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1_000)); + assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(1), 2_000).is_committed()); + assert!(leases + .activate_access_lease( + &lease_id, + &session, + 2_000, + crate::HlsAccessLeaseTiming { active_window_ms: 10, valid_window_ms: 60_000 } + ) + .is_activated()); + let identity = leases.response_snapshot(&lease_id, &session, 2_000).unwrap().media_identity().unwrap(); + assert!( + leases.terminal_playback_snapshots_for_session(&session, 2_011).is_empty(), + "unstarted lease after active window" + ); + let token = + leases.record_segment_request_started_if_identity_matches(&lease_id, &session, identity, 40, 2_001).unwrap(); + assert_eq!( + leases.record_segment_transfer_started_if_identity_matches(&lease_id, &session, identity, token, 2_002), + Some(true) + ); + assert_eq!(leases.active_live_playback_snapshots_for_session(&session, 2_011), Vec::new()); + leases.by_lease_id.get_mut(&lease_id).unwrap().state = HlsAccessLeaseState::Idle; + let monitored = leases.terminal_playback_snapshots_for_session(&session, 2_011); + assert_eq!(monitored.len(), 1); + assert_eq!(monitored[0].state, HlsAccessLeaseState::Idle); + assert_eq!(leases.terminal_playback_snapshots_for_session(&session, 70_000), Vec::new()); +} + pub(in crate::lease::tests) fn manifest_snapshot_for_route( source_rendered_at_ms: u64, proxy_session_id: &ProxySessionId, @@ -87,6 +135,15 @@ pub(in crate::lease::tests) fn terminal_plan_at( ) -> Arc { let mut base_manifest = manifest_snapshot_for_route(1, proxy_session_id, lease_id); base_manifest.snapshot_generation = 1; + terminal_plan_with_manifest(generation, base_manifest, created_at_ms, false) +} + +fn terminal_plan_with_manifest( + generation: u64, + base_manifest: HlsLeaseManifestSnapshot, + created_at_ms: u64, + unavailable_prefix: bool, +) -> Arc { let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); let asset = snapshot_terminal_media_asset(&transport_stream).expect("terminal asset"); let expected_asset = @@ -96,7 +153,11 @@ pub(in crate::lease::tests) fn terminal_plan_at( .visible_proxy_seqs() .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { proxy_seq, - media_state: HlsTerminalBaseMediaState::Ready, + media_state: if unavailable_prefix && proxy_seq != base_manifest.last_proxy_seq { + HlsTerminalBaseMediaState::NotReady + } else { + HlsTerminalBaseMediaState::Ready + }, required_map_ready: true, required_key_ready: true, protection: HlsTerminalBaseProtection::Protectable, @@ -704,3 +765,34 @@ fn late_segment_completion_after_terminal_transition_does_not_advance_cursor() { assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); } + +#[test] +fn terminal_suffix_keeps_last_live_identity_when_discontinuity_moves_into_the_sequence() { + let session = ProxySessionId("discontinuity-splice".into()); + let id = HlsAccessLeaseId("discontinuity-splice".into()); + let mut store = HlsAccessLeaseStore::default(); + store.prepare_access_lease(lease(id.clone(), &session.0, 1_000)); + let mut manifest = manifest_snapshot(1); + Arc::make_mut(&mut manifest.visible_segments).last_mut().unwrap().discontinuity_before = true; + assert!(publish_manifest_snapshot(&mut store, &id, &session, manifest, 2_000).is_committed()); + let preparation = prepare_terminal(&mut store, &id, &session, 4, Some(2_000)); + let plan = terminal_plan_with_manifest( + preparation.decision_generation, + preparation.manifest_snapshot.clone(), + 3_000, + true, + ); + assert_eq!(plan.base_manifest.visible_segments.len(), 1); + assert!(!plan.base_manifest.visible_segments[0].discontinuity_before); + assert_eq!(plan.base_manifest.discontinuity_sequence, preparation.manifest_snapshot.discontinuity_sequence + 1); + let mut wrong = (*plan).clone(); + Arc::make_mut(&mut wrong.base_manifest.visible_segments)[0].uri = "/wrong-live.ts".into(); + assert_eq!( + store.commit_terminal_tail_if_generation_matches(&id, &session, &preparation, 3_000, Arc::new(wrong)), + HlsTerminalCommitOutcome::SupersededGeneration + ); + assert_eq!( + store.commit_terminal_tail_if_generation_matches(&id, &session, &preparation, 3_000, plan), + HlsTerminalCommitOutcome::Committed + ); +} diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index e017645dc..46a80bbec 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -239,7 +239,7 @@ pub mod api { #[cfg(test)] pub(crate) use self::timeline::TimelineMapError; pub(crate) use self::{ - availability::{hls_recovery_timing_policy, HlsManifestAcceptanceDirective, HLS_PLAYBACK_RATE_GUARD_MILLI}, + availability::{hls_recovery_timing_policy, HlsManifestAcceptanceDirective}, availability_reevaluation::HlsAvailabilityReevaluationRegistration, backpressure::{classify_hls_backpressure, HlsBackpressureState}, cache::{ diff --git a/backend/hls/src/live_window.rs b/backend/hls/src/live_window.rs index 3d0fdddbc..3a60d0ef0 100644 --- a/backend/hls/src/live_window.rs +++ b/backend/hls/src/live_window.rs @@ -158,11 +158,16 @@ impl HlsSession { pub(crate) fn record_live_window_publication( &mut self, lease_id: &crate::HlsAccessLeaseId, + issued_at_ms: u64, snapshot: &crate::media_reserve::HlsLeaseManifestSnapshot, ) { - if let Some(demand) = self.live_window_demands.get_mut(lease_id) { - demand.published = snapshot.visible_proxy_seqs().collect(); - } + let demand = self.live_window_demands.entry(lease_id.clone()).or_insert_with(|| HlsLiveWindowDemand { + issued_at_ms, + view: HlsLiveWindowView::Entry, + published: Vec::new(), + }); + demand.issued_at_ms = issued_at_ms; + demand.published = snapshot.visible_proxy_seqs().collect(); self.longest_rendered_playlist_duration_ms = self.longest_rendered_playlist_duration_ms.max(snapshot.playlist_duration_ms); for seq in snapshot.visible_proxy_seqs() { diff --git a/backend/hls/src/manager/activity.rs b/backend/hls/src/manager/activity.rs index a703033e4..fc4d7c4de 100644 --- a/backend/hls/src/manager/activity.rs +++ b/backend/hls/src/manager/activity.rs @@ -8,7 +8,7 @@ use std::sync::Arc; #[derive(Debug, Clone, Copy)] enum HlsMediaActivityCommitKind { Access, - Delivered, + Delivered(Option), LiveSegmentCompletion(HlsPlaybackRequestToken), } @@ -89,6 +89,7 @@ impl HlsProxyManager { lease_id: &HlsAccessLeaseId, proxy_session_id: &ProxySessionId, lease_identity: HlsMediaLeaseIdentity, + token: Option, now_ms: u64, ) -> HlsMediaActivityCommitOutcome { self.commit_media_activity_if_identity_matches( @@ -97,7 +98,7 @@ impl HlsProxyManager { proxy_session_id, lease_identity, now_ms, - HlsMediaActivityCommitKind::Delivered, + HlsMediaActivityCommitKind::Delivered(token), ) .await } @@ -243,9 +244,19 @@ impl HlsProxyManager { }; } let (current, capacity_protection_released) = match kind { - HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered => { + HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered(None) => { (leases.media_identity_is_current(lease_id, proxy_session_id, lease_identity, now_ms), false) } + HlsMediaActivityCommitKind::Delivered(Some(token)) => { + let started = leases.record_segment_transfer_started_if_identity_matches( + lease_id, + proxy_session_id, + lease_identity, + token, + now_ms, + ); + (started.is_some(), started == Some(true)) + } HlsMediaActivityCommitKind::LiveSegmentCompletion(token) => { let completion = leases.record_segment_request_completed_if_identity_matches( lease_id, @@ -264,7 +275,7 @@ impl HlsProxyManager { }; } session.mark_authorized_media_access(now_ms); - if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered) { + if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered(_)) { session.activity.last_delivered_media_at_ms = Some(now_ms); } HlsMediaActivityCommitAttempt::Completed { diff --git a/backend/hls/src/manager/lifecycle.rs b/backend/hls/src/manager/lifecycle.rs index 1a9df6db3..cc2635d82 100644 --- a/backend/hls/src/manager/lifecycle.rs +++ b/backend/hls/src/manager/lifecycle.rs @@ -16,6 +16,13 @@ pub(super) fn hls_session_idle_protection_retry_at(due_at_ms: u64, now_ms: u64) } impl HlsProxyManager { + pub(crate) async fn terminal_playback_snapshots_for_session( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Vec { + self.access_leases.write().await.terminal_playback_snapshots_for_session(proxy_session_id, now_ms) + } pub fn lifecycle(&self) -> &Arc { &self.lifecycle } pub async fn active_live_playback_snapshots_for_session( diff --git a/backend/hls/src/manager/publication.rs b/backend/hls/src/manager/publication.rs index 2bcb0b64b..a78d0f59f 100644 --- a/backend/hls/src/manager/publication.rs +++ b/backend/hls/src/manager/publication.rs @@ -525,7 +525,12 @@ impl HlsProxyManager { } else { published_transient_resource_ids }; - return leases.commit_manifest_publication_with_resources( + let mut session_guard = match session.as_ref() { + Some(session) => Some(session.write().await), + None => None, + }; + let published = snapshot.clone(); + let outcome = leases.commit_manifest_publication_with_resources( lease_id, proxy_session_id, expected, @@ -533,6 +538,12 @@ impl HlsProxyManager { published_transient_resource_ids, now_ms, ); + if outcome.is_committed() { + if let Some(session) = session_guard.as_mut() { + session.record_live_window_publication(lease_id, lease_issued_at_ms, &published); + } + } + return outcome; }; let Some(current_session) = self.sessions.hold_current_proxy_session(proxy_session_id).await else { return HlsLeaseManifestPublicationOutcome::Rejected( @@ -639,7 +650,6 @@ impl HlsProxyManager { cursor: &lease.playback_cursor, ready_timeline: &ready_timeline, now_ms, - playback_rate_guard_milli: super::super::HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0), origin_path_degraded: true, recovery_committed: false, diff --git a/backend/hls/src/manager/revision_recovery.rs b/backend/hls/src/manager/revision_recovery.rs index 985be2727..87a55537a 100644 --- a/backend/hls/src/manager/revision_recovery.rs +++ b/backend/hls/src/manager/revision_recovery.rs @@ -230,7 +230,7 @@ impl HlsProxyManager { .commit_manifest_publication_with_resources(lease_id, session_id, expected, snapshot, resources, now_ms); if outcome.is_committed() { if let Some(snapshot) = published { - session.record_live_window_publication(lease_id, &snapshot); + session.record_live_window_publication(lease_id, expected.lease_issued_at_ms(), &snapshot); } } outcome diff --git a/backend/hls/src/manager/terminal.rs b/backend/hls/src/manager/terminal.rs index 73187ec70..84020452b 100644 --- a/backend/hls/src/manager/terminal.rs +++ b/backend/hls/src/manager/terminal.rs @@ -64,6 +64,11 @@ fn evaluate_terminal_commit_authorization( now_ms: u64, ) -> HlsTerminalCommitAuthorization { let recovery_snapshot = hls_acceptance_recovery_snapshot(session, now_ms); + if !preparation.trigger.is_runtime_policy() + && (!preparation.reserve.is_exhausted() || !session.origin_control.path_is_degraded_at(now_ms)) + { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); + } // Acceptance lifecycle generations describe recovery evidence, not media // continuity. A later episode must not invalidate a cutover snapshot while // the media progress, readiness, epoch, and lease generations guarded below @@ -71,21 +76,10 @@ fn evaluate_terminal_commit_authorization( if !terminal_media_requirement_is_bound_to_preparation(preparation) { return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); } - if let HlsTerminalLeaseDecision::CustomResponse(plan) = decision { - if preparation.trigger.is_runtime_policy() - || !plan.matches_route(&session.proxy_session_id, lease_id) - || plan.generation.0 != preparation.decision_generation - || plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation - { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); - } - if !session.origin_control.path_is_degraded_at(now_ms) { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); - } - if now_ms < preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); - } - return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; + if matches!(decision, HlsTerminalLeaseDecision::CustomResponse(_)) { + // An ongoing lease must retain its exact live splice base, even after + // the old deadline; a standalone custom playlist cannot do that. + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); } let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_)) && !session.can_install_terminal_tail_protection(lease_id); @@ -604,6 +598,25 @@ impl HlsProxyManager { if preparation.trigger.is_runtime_policy() { return None; } + if let HlsTerminalLeaseDecision::Tail(plan) = &command.decision { + if preparation.manifest_snapshot.startup_revisions.is_some() + && plan.base_manifest.startup_revisions.is_none() + { + return Some(HlsTerminalCommitOutcome::BundleIncompatible); + } + if let Some(publication) = &plan.base_manifest.startup_revisions { + let seq = preparation.manifest_snapshot.last_proxy_seq; + let Some(bound) = publication.revisions.get(&seq) else { + return Some(HlsTerminalCommitOutcome::BundleIncompatible); + }; + if leases.published_segment_revision(lease_id, seq).as_ref() != Some(bound) { + return Some(HlsTerminalCommitOutcome::SupersededGeneration); + } + if !matches!(bound.revision().state(), crate::SegmentRevisionState::Complete(_)) { + return Some(HlsTerminalCommitOutcome::BundleIncompatible); + } + } + } if session.origin_control.progress_generation == preparation.origin_progress_generation && session.activity.media_readiness_generation == preparation.media_readiness_generation && session.origin_control.origin_epoch == preparation.origin_epoch @@ -667,7 +680,7 @@ impl HlsProxyManager { proxy_session_id, preparation, now_ms, - Arc::clone(plan), + plan, ), (HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches( lease_id, @@ -710,6 +723,7 @@ impl HlsProxyManager { } return outcome; } + session.live_window_demands.remove(lease_id); Self::finish_terminal_commit(leases, session, proxy_session_id); HlsTerminalCommitOutcome::Committed } @@ -744,6 +758,7 @@ impl HlsProxyManager { session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); return outcome; } + session.live_window_demands.remove(lease_id); Self::finish_terminal_commit(leases, session, proxy_session_id); HlsTerminalCommitOutcome::Committed } diff --git a/backend/hls/src/manager/tests.rs b/backend/hls/src/manager/tests.rs index 58cad7626..d18839eb3 100644 --- a/backend/hls/src/manager/tests.rs +++ b/backend/hls/src/manager/tests.rs @@ -9,8 +9,7 @@ use super::{ recovery_timing::{ HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, - HlsRecoveryWorkload, HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow, - HlsTerminalMediaPreparationState, HlsTransitionMarginMs, + HlsRecoveryWorkload, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, HlsTransitionMarginMs, }, runtime_custom_tail::HlsRuntimeCustomTailAssetIdentity, session::{HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY}, diff --git a/backend/hls/src/manager/tests/support.rs b/backend/hls/src/manager/tests/support.rs index 0e539b85d..376e6b122 100644 --- a/backend/hls/src/manager/tests/support.rs +++ b/backend/hls/src/manager/tests/support.rs @@ -2,10 +2,9 @@ use super::{ prepared_terminal_bundle_key, snapshot_terminal_media_asset, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, HlsManifestAcceptanceGeneration, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsProxyManager, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, - HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, - HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome, - HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, - HlsTerminalTailPlan, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs, + HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, HlsTerminalCommitMediaGuard, + HlsTerminalCommitOutcome, HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, + HlsTerminalMediaPreparationState, HlsTerminalTailPlan, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs, }; use crate::{ build_terminal_tail_plan, @@ -31,19 +30,18 @@ pub(in crate::manager::tests) const TERMINAL_ASSET_BYTES: &[u8] = pub(in crate::manager::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot { let transition_margin = HlsTransitionMarginMs::from_millis(12_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + let guaranteed_reserve_ms = 0; HlsLeaseReserveSnapshot { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: 12_000_u64.saturating_add(guaranteed_reserve_ms), conservative_playback_position_ms: 12_000, + playback_position_known: true, guaranteed_reserve_ms, initial_hidden_ready_duration_ms: 0, transition_margin, key_readiness_valid_until_ms: None, recovery_required: true, - cutover_required: false, + cutover_required: true, } } @@ -163,6 +161,10 @@ pub(in crate::manager::tests) async fn publish_manifest_snapshot( snapshot: HlsLeaseManifestSnapshot, now_ms: u64, ) -> bool { + let mut snapshot = snapshot; + for segment in Arc::make_mut(&mut snapshot.visible_segments) { + segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into(); + } let Some(guard) = manager.prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms).await else { return false; @@ -312,6 +314,7 @@ pub(in crate::manager::tests) fn complete_failed_acceptance_episode( session: &mut super::super::super::HlsSession, now_ms: u64, ) -> u64 { + session.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure; let generation = begin_test_acceptance_episode(session, now_ms); let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); assert_eq!(episode.generation, generation); diff --git a/backend/hls/src/manager/tests/terminal.rs b/backend/hls/src/manager/tests/terminal.rs index ac6c31c84..d0d9087e3 100644 --- a/backend/hls/src/manager/tests/terminal.rs +++ b/backend/hls/src/manager/tests/terminal.rs @@ -5,11 +5,11 @@ use super::{ spawn_terminal_commit_retry_worker, terminal_plan, terminal_preparation_request, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsLeaseCutoverTiming, HlsMediaActivityCommitOutcome, HlsProxyManager, HlsRecoveryExecutionState, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, - HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitCommand, HlsTerminalCommitMediaGuard, - HlsTerminalCommitOutcome, HlsTerminalCommitOwnerKey, HlsTerminalCommitPayload, HlsTerminalCommitRequest, - HlsTerminalCommitRetryDecision, HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision, - HlsTerminalCommitWindow, HlsTerminalLeaseDecision, HlsTerminalMediaRequirementSource, - HlsTerminalTailPreparationRequest, HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY, + HlsTerminalCommitCommand, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome, HlsTerminalCommitOwnerKey, + HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitRetryDecision, + HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision, HlsTerminalCommitWindow, + HlsTerminalLeaseDecision, HlsTerminalMediaRequirementSource, HlsTerminalTailPreparationRequest, + HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY, }; use crate::{ media_reserve::HlsLeaseReserveSnapshot, terminal_tail::HlsLeasePlaybackMode, HlsAccessLeaseId, HlsAccessLeaseState, @@ -155,6 +155,7 @@ async fn hls_terminal_commit_missing_acceptance_exact_ready_bundle_commits_tail( } ); + session.write().await.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure; let outcome = commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan); assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); @@ -495,7 +496,7 @@ async fn hls_terminal_commit_submission_delayed_stale_incoming_preserves_authori let HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } = next_terminal_commit_retry( 1, 2_000, - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), + 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms(), ) else { panic!("fixture has retry budget"); }; @@ -606,11 +607,14 @@ async fn hls_terminal_commit_submission_newer_command_replaces_lock_busy_owner() async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() { let (manager, session, proxy_session_id, lease_id, preparation, plan) = prepared_terminal_commit_fixture("retry-deadline").await; - let after_deadline_ms = - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch().saturating_add(1); - manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms); + let after_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms() + 1; let session_guard = session.write().await; + assert!(matches!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, Arc::clone(&plan)), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms); assert_eq!( commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed @@ -630,15 +634,15 @@ async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() { } #[tokio::test] -async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_deadline() { +async fn hls_terminal_commit_starts_after_old_deadline_but_retry_budget_is_exclusive() { let (manager, session, proxy_session_id, lease_id, preparation, plan) = prepared_terminal_commit_fixture("initial-exclusive-deadline").await; - let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let safe_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms(); manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms); assert_eq!( commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed + HlsTerminalCommitOutcome::Committed ); assert!(matches!( manager @@ -646,13 +650,13 @@ async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_de .await .expect("exclusive deadline leaves the initial-attempt lease live") .playback_mode, - HlsLeasePlaybackMode::Live + HlsLeasePlaybackMode::TerminalTail(_) )); - assert!(!session.read().await.has_terminal_tail_protections()); + assert!(session.read().await.has_terminal_tail_protections()); let (manager, session, proxy_session_id, lease_id, preparation, plan) = prepared_terminal_commit_fixture("retry-exclusive-deadline").await; - let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let safe_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms(); let session_guard = session.write().await; assert!(matches!( commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), @@ -916,7 +920,7 @@ async fn hls_terminal_commit_lease_end_prevents_commit_and_gc_protection() { complete_failed_acceptance_episode(&mut session, 2_000) }; let mut preparation_request = terminal_preparation_request(&lease_id, &proxy_session_id, progress_generation); - preparation_request.reserve.guaranteed_reserve_ms = 100_000; + preparation_request.reserve.guaranteed_reserve_ms = 0; preparation_request.reserve.guaranteed_media_horizon_ms = preparation_request .reserve .conservative_playback_position_ms @@ -1003,6 +1007,7 @@ async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farthe let far_reserve = HlsLeaseReserveSnapshot { guaranteed_media_horizon_ms: 36_000, conservative_playback_position_ms: 12_000, + playback_position_known: true, guaranteed_reserve_ms: 24_000, cutover_required: false, ..cutover_reserve() @@ -1037,10 +1042,7 @@ async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farthe )) .await .expect("terminal preparation"); - assert_eq!( - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), - 2_000_u64.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()) - ); + assert_eq!(preparation.cutover_timing.lease_exhaustion_at.as_millis_since_epoch(), 2_000); assert_eq!(far_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 14_000); let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &terminal_lease_id); let expected_protection = Arc::clone(&plan.protected_base_proxy_seqs); @@ -1285,3 +1287,27 @@ async fn unavailable_after_expired_splice_deadline_still_rejects_recovered_origi HlsLeasePlaybackMode::Live ); } + +#[tokio::test] +async fn final_terminal_authorization_rejects_every_positive_reserve_and_unknown_zero() { + let (manager, session, proxy, lease, preparation, plan) = prepared_terminal_commit_fixture("final-zero-gate").await; + let margin = preparation.reserve.transition_margin.as_millis(); + let acquisition = crate::recovery_timing::HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis(); + for (reserve_ms, known) in [(margin + acquisition, true), (margin, true), (1, true), (0, false)] { + let mut early = preparation.clone(); + early.reserve.guaranteed_reserve_ms = reserve_ms; + early.reserve.playback_position_known = known; + assert_eq!( + commit_terminal_plan(&manager, &session, &lease, &proxy, &early, 2_000, Arc::clone(&plan)), + HlsTerminalCommitOutcome::CutoverNoLongerRequired + ); + assert_eq!( + manager.access_lease_response_snapshot(&lease, &proxy, 2_000).await.unwrap().playback_mode, + HlsLeasePlaybackMode::Live + ); + } + assert_eq!( + commit_terminal_plan(&manager, &session, &lease, &proxy, &preparation, 2_000, plan), + HlsTerminalCommitOutcome::Committed + ); +} diff --git a/backend/hls/src/manifest_acceptance/tests/episode.rs b/backend/hls/src/manifest_acceptance/tests/episode.rs index 0a1b97c0c..ced0dad57 100644 --- a/backend/hls/src/manifest_acceptance/tests/episode.rs +++ b/backend/hls/src/manifest_acceptance/tests/episode.rs @@ -256,7 +256,7 @@ fn hls_recovery_timing_candidate_binding_and_staging_reduce_eta_without_moving_d episode .estimated_recovery_completion_at(generation, 5_000) .map(HlsEstimatedRecoveryCompletionAtMs::as_millis_since_epoch), - staged_eta.map(|eta| 5_000_u64.saturating_add(eta)) + staged_eta.map(|eta| 5_000_u64.saturating_add(eta).min(frozen_deadline.as_millis_since_epoch())) ); assert_eq!(episode.remaining_recovery_workload(HlsManifestAcceptanceGeneration(5)), None); diff --git a/backend/hls/src/manifest_fetch/episode.rs b/backend/hls/src/manifest_fetch/episode.rs index b25bc7bce..93cec7273 100644 --- a/backend/hls/src/manifest_fetch/episode.rs +++ b/backend/hls/src/manifest_fetch/episode.rs @@ -118,7 +118,7 @@ fn acceptance_episode_timing( .origin_control .target_duration_snapshot_ms .or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000))) - .unwrap_or(15_000); + .unwrap_or_else(|| context.acceptance_timing_seed.map_or(15_000, |seed| seed.target_duration_ms)); let seed = context.acceptance_timing_seed.unwrap_or(HlsAcceptanceEpisodeTimingSeed { target_duration_ms: fallback_target_duration_ms, transition_margin: HlsTransitionMarginMs::from_millis(fallback_target_duration_ms), @@ -129,7 +129,7 @@ fn acceptance_episode_timing( HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { started_at_ms, burst_plan, - target_duration_ms: seed.target_duration_ms, + target_duration_ms: fallback_target_duration_ms, transition_margin: seed.transition_margin, workload: seed.workload, observed_latency: session.origin_control.recovery_samples.latency_snapshot(), diff --git a/backend/hls/src/media_reserve.rs b/backend/hls/src/media_reserve.rs index d82b3e423..4e0ca8467 100644 --- a/backend/hls/src/media_reserve.rs +++ b/backend/hls/src/media_reserve.rs @@ -113,6 +113,8 @@ impl HlsLeaseManifestSnapshot { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct HlsLeasePlaybackCursor { pub first_requested_proxy_seq: Option, + pub first_segment_started_at_ms: Option, + pub first_segment_started_proxy_seq: Option, pub first_segment_completed_at_ms: Option, pub highest_contiguous_completed_proxy_seq: Option, pub last_requested_proxy_seq: Option, @@ -121,6 +123,7 @@ pub struct HlsLeasePlaybackCursor { request_epoch: u64, highest_requested_proxy_seq: Option, first_requested_media_start_ms: Option, + first_segment_media_start_ms: Option, highest_contiguous_completed_media_end_ms: Option, out_of_order_completed_proxy_seqs: BTreeMap>, } @@ -134,6 +137,7 @@ pub struct HlsPlaybackRequestToken { proxy_seq: u64, request_epoch: u64, media_end_ms: Option, + media_start_ms: Option, } /// Result of committing a fully delivered segment to a playback cursor. @@ -170,6 +174,9 @@ impl HlsLeasePlaybackCursor { if forward_seek { self.request_epoch = self.request_epoch.saturating_add(1); self.first_requested_proxy_seq = Some(proxy_seq); + self.first_segment_started_at_ms = None; + self.first_segment_started_proxy_seq = None; + self.first_segment_media_start_ms = None; self.first_segment_completed_at_ms = None; self.highest_contiguous_completed_proxy_seq = None; self.highest_requested_proxy_seq = Some(proxy_seq); @@ -188,7 +195,12 @@ impl HlsLeasePlaybackCursor { ); self.cursor_generation = self.cursor_generation.saturating_add(1); - HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch, media_end_ms: None } + HlsPlaybackRequestToken { + proxy_seq, + request_epoch: self.request_epoch, + media_end_ms: None, + media_start_ms: None, + } } /// Binds playback evidence to the immutable session media clock before cache entries can be collected. @@ -203,9 +215,25 @@ impl HlsLeasePlaybackCursor { self.first_requested_media_start_ms.get_or_insert(segment.media_start_ms); } token.media_end_ms = Some(segment.media_start_ms.saturating_add(segment.duration_ms)); + token.media_start_ms = Some(segment.media_start_ms); token } + /// Starts the 1x media clock only when the first request actually yields bytes. + /// A successor or duplicate cannot reset it; tokens from earlier seeks are rejected. + pub fn record_transfer_started(&mut self, token: HlsPlaybackRequestToken, started_at_ms: u64) -> bool { + if token.request_epoch != self.request_epoch || self.first_requested_proxy_seq.is_none() { + return false; + } + if self.first_segment_started_at_ms.is_none() { + self.first_segment_started_at_ms = Some(started_at_ms); + self.first_segment_started_proxy_seq = Some(token.proxy_seq); + self.first_segment_media_start_ms = token.media_start_ms; + self.cursor_generation = self.cursor_generation.saturating_add(1); + } + true + } + /// Commits a token only after the corresponding response body was delivered completely. pub fn record_request_completed( &mut self, @@ -295,6 +323,7 @@ pub struct HlsLeaseReserveSnapshot { pub availability_basis: HlsLeaseReserveAvailabilityBasis, pub guaranteed_media_horizon_ms: u64, pub conservative_playback_position_ms: u64, + pub playback_position_known: bool, pub guaranteed_reserve_ms: u64, pub initial_hidden_ready_duration_ms: u64, pub transition_margin: HlsTransitionMarginMs, @@ -303,6 +332,11 @@ pub struct HlsLeaseReserveSnapshot { pub cutover_required: bool, } +impl HlsLeaseReserveSnapshot { + /// Missing playback evidence is not proof that the player consumed its reserve. + pub fn is_exhausted(self) -> bool { self.playback_position_known && self.guaranteed_reserve_ms == 0 } +} + /// Typed source of guaranteed reserve evidence. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HlsLeaseReserveAvailabilityBasis { @@ -316,7 +350,6 @@ pub struct HlsLeaseReserveInput<'a> { pub cursor: &'a HlsLeasePlaybackCursor, pub ready_timeline: &'a HlsReadyTimelineSnapshot, pub now_ms: u64, - pub playback_rate_guard_milli: u16, pub recovery_trigger_budget: HlsRecoveryTriggerBudgetMs, pub origin_path_degraded: bool, pub recovery_committed: bool, @@ -329,6 +362,7 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv availability_basis: HlsLeaseReserveAvailabilityBasis::TransientOriginBacked, guaranteed_media_horizon_ms: 0, conservative_playback_position_ms: 0, + playback_position_known: false, guaranteed_reserve_ms: 0, initial_hidden_ready_duration_ms: 0, transition_margin, @@ -344,9 +378,18 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv }; let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest, first_needed_seq); let visible_tail = aligned_visible_tail_ms(input.ready_timeline, input.manifest); - let guaranteed_media_horizon_ms = guaranteed_ready_span.horizon_ms; - let conservative_playback_position_ms = - conservative_playback_position(&input, visible_tail, guaranteed_media_horizon_ms); + // Fully delivered contiguous history survives metadata GC; incomplete bodies do not advance it. + let guaranteed_media_horizon_ms = + guaranteed_ready_span.horizon_ms.max(input.cursor.highest_contiguous_completed_media_end_ms.unwrap_or(0)); + let position = conservative_playback_position(&input); + let playback_position_known = position.is_some(); + let conservative_playback_position_ms = position.unwrap_or_else(|| { + input + .cursor + .first_requested_proxy_seq + .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) + .map_or(visible_tail, |unit| visible_tail.max(unit.start_ms)) + }); let guaranteed_reserve_ms = guaranteed_media_horizon_ms.saturating_sub(conservative_playback_position_ms); let initial_hidden_ready_duration_ms = guaranteed_media_horizon_ms.saturating_sub(visible_tail); let transition_margin = HlsTransitionMarginMs::from_millis( @@ -363,6 +406,7 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms, conservative_playback_position_ms, + playback_position_known, guaranteed_reserve_ms, initial_hidden_ready_duration_ms, transition_margin, @@ -461,37 +505,13 @@ fn aligned_visible_tail_ms(timeline: &HlsReadyTimelineSnapshot, manifest: &HlsLe .map_or(manifest.last_visible_media_end_ms, |unit| unit.start_ms.saturating_add(manifest.playlist_duration_ms)) } -fn conservative_playback_position( - input: &HlsLeaseReserveInput<'_>, - visible_tail_ms: u64, - guaranteed_media_horizon_ms: u64, -) -> u64 { - let Some(first_seq) = input.cursor.first_requested_proxy_seq else { - return visible_tail_ms; - }; - let Some(first_start_ms) = input.cursor.first_requested_media_start_ms.or_else(|| { +fn conservative_playback_position(input: &HlsLeaseReserveInput<'_>) -> Option { + let started_at_ms = input.cursor.first_segment_started_at_ms?; + let first_seq = input.cursor.first_segment_started_proxy_seq?; + let first_start_ms = input.cursor.first_segment_media_start_ms.or_else(|| { input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq).map(|unit| unit.start_ms) - }) else { - return visible_tail_ms.max(guaranteed_media_horizon_ms); - }; - if input.cursor.first_segment_completed_at_ms.is_none() { - return visible_tail_ms.max(first_start_ms); - } - let delivered_end = input - .cursor - .highest_contiguous_completed_media_end_ms - .or_else(|| { - input - .cursor - .highest_contiguous_completed_proxy_seq - .and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq)) - .map(|unit| unit.start_ms.saturating_add(unit.duration_ms)) - }) - .unwrap_or(guaranteed_media_horizon_ms); - let elapsed_guarded = input.cursor.first_segment_completed_at_ms.map_or(0, |completed_at_ms| { - input.now_ms.saturating_sub(completed_at_ms).saturating_mul(u64::from(input.playback_rate_guard_milli)) / 1_000 - }); - delivered_end.min(first_start_ms.saturating_add(elapsed_guarded)) + })?; + Some(first_start_ms.saturating_add(input.now_ms.saturating_sub(started_at_ms))) } fn guaranteed_transition_margin( @@ -629,6 +649,7 @@ pub fn evaluate_startup_admission(input: HlsStartupAdmissionInput<'_>) -> HlsSta #[cfg(test)] mod tests { + mod playback; use super::*; fn manifest() -> HlsLeaseManifestSnapshot { @@ -714,7 +735,6 @@ mod tests { cursor, ready_timeline, now_ms, - playback_rate_guard_milli: 1_050, recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000), origin_path_degraded: true, recovery_committed: false, @@ -874,17 +894,19 @@ mod tests { fn completed_first_segment_uses_measured_start_position() { let mut cursor = HlsLeasePlaybackCursor::default(); let request = cursor.record_request_started(1, 1_000); + cursor.record_transfer_started(request, 1_000); assert_eq!(cursor.record_request_completed(request, 1_000), HlsPlaybackCompletionOutcome::Advanced); let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 3_000); - assert_eq!(reserve.conservative_playback_position_ms, 6_100); - assert_eq!(reserve.guaranteed_reserve_ms, 20_900); + assert_eq!(reserve.conservative_playback_position_ms, 6_000); + assert_eq!(reserve.guaranteed_reserve_ms, 21_000); } #[test] fn sliding_manifest_keeps_cursor_and_ready_horizon_on_one_timeline() { let mut cursor = HlsLeasePlaybackCursor::default(); let request = cursor.record_request_started(0, 100); + cursor.record_transfer_started(request, 100); assert_eq!(cursor.record_request_completed(request, 100), HlsPlaybackCompletionOutcome::Advanced); let mut current_manifest = manifest(); current_manifest.first_proxy_seq = 2; @@ -911,12 +933,12 @@ mod tests { current_manifest.last_visible_media_end_ms = 9_000; assert_eq!(cursor.ready_timeline_start_proxy_seq(current_manifest.first_proxy_seq), 0); - let reserve = reserve(¤t_manifest, &cursor, &ready_timeline(), 2_100); + let reserve = reserve(¤t_manifest, &cursor, &ready_timeline(), 2_000); - assert_eq!(reserve.conservative_playback_position_ms, 2_100); + assert_eq!(reserve.conservative_playback_position_ms, 1_900); assert_eq!(reserve.guaranteed_media_horizon_ms, 27_000); assert_eq!(reserve.initial_hidden_ready_duration_ms, 9_000); - assert_eq!(reserve.guaranteed_reserve_ms, 24_900); + assert_eq!(reserve.guaranteed_reserve_ms, 25_100); assert!(!reserve.cutover_required); } @@ -940,10 +962,14 @@ mod tests { let second = cursor.record_request_started(1, 200); let third = cursor.record_request_started(2, 300); + cursor.record_transfer_started(third, 600); + assert_eq!(cursor.record_request_completed(third, 600), HlsPlaybackCompletionOutcome::Buffered); assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None); + cursor.record_transfer_started(first, 700); assert_eq!(cursor.record_request_completed(first, 700), HlsPlaybackCompletionOutcome::Advanced); assert_eq!(cursor.highest_contiguous_completed_proxy_seq, Some(0)); + cursor.record_transfer_started(second, 800); assert_eq!(cursor.record_request_completed(second, 800), HlsPlaybackCompletionOutcome::Advanced); assert_eq!(cursor.highest_contiguous_completed_proxy_seq, Some(2)); assert_eq!(cursor.first_segment_completed_at_ms, Some(700)); @@ -955,19 +981,23 @@ mod tests { let old_request = cursor.record_request_started(0, 100); let seek_request = cursor.record_request_started(4, 200); + cursor.record_transfer_started(old_request, 300); + assert_eq!(cursor.record_request_completed(old_request, 300), HlsPlaybackCompletionOutcome::StaleRequest); assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None); + cursor.record_transfer_started(seek_request, 400); assert_eq!(cursor.record_request_completed(seek_request, 400), HlsPlaybackCompletionOutcome::Advanced); let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 1_400); - assert_eq!(reserve.conservative_playback_position_ms, 19_050); - assert_eq!(reserve.guaranteed_reserve_ms, 7_950); + assert_eq!(reserve.conservative_playback_position_ms, 19_000); + assert_eq!(reserve.guaranteed_reserve_ms, 8_000); } #[test] fn regressing_clock_saturates_playback_estimate() { let mut cursor = HlsLeasePlaybackCursor::default(); let request = cursor.record_request_started(1, 2_000); + cursor.record_transfer_started(request, 2_000); assert_eq!(cursor.record_request_completed(request, 2_000), HlsPlaybackCompletionOutcome::Advanced); let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 1_000); @@ -1309,9 +1339,13 @@ mod tests { cursor.bind_request_timing(token, &session.segments[&seq]) }) .collect::>(); + cursor.record_transfer_started(tokens[0], 1_000); + cursor.record_transfer_started(tokens[2], 1_100); assert_eq!(cursor.record_request_completed(tokens[2], 1_100), HlsPlaybackCompletionOutcome::Buffered); + cursor.record_transfer_started(tokens[0], 1_200); assert_eq!(cursor.record_request_completed(tokens[0], 1_200), HlsPlaybackCompletionOutcome::Advanced); assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(4_000)); + cursor.record_transfer_started(tokens[1], 1_300); assert_eq!(cursor.record_request_completed(tokens[1], 1_300), HlsPlaybackCompletionOutcome::Advanced); assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(15_000)); let mut current = manifest(); @@ -1330,8 +1364,8 @@ mod tests { 11_200, ); assert_eq!(before, after); - assert_eq!(after.conservative_playback_position_ms, 10_500); - assert_eq!(after.guaranteed_reserve_ms, 16_500); + assert_eq!(after.conservative_playback_position_ms, 10_200); + assert_eq!(after.guaranteed_reserve_ms, 16_800); } #[test] @@ -1340,6 +1374,7 @@ mod tests { let mut cursor = HlsLeasePlaybackCursor::default(); let first = cursor.record_request_started(0, 1_000); let first = cursor.bind_request_timing(first, &session.segments[&0]); + cursor.record_transfer_started(first, 1_000); assert_eq!(cursor.record_request_completed(first, 1_000), HlsPlaybackCompletionOutcome::Advanced); let mut current = manifest(); current.first_proxy_seq = 2; @@ -1350,13 +1385,13 @@ mod tests { assert_eq!(cursor.ready_timeline_start_proxy_seq(2), 1); session.segments.remove(&0); let timeline = session.ready_timeline_snapshot(1, 2_000); - assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 25_950); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 26_000); session.segments.get_mut(&1).expect("undelivered segment").status = crate::SegmentCacheStatus::Discovered; let timeline = session.ready_timeline_snapshot(1, 2_000); - assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 2_950); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 3_000); session.segments.remove(&1); let timeline = session.ready_timeline_snapshot(1, 2_000); - assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 0); + assert_eq!(reserve(¤t, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 3_000); } #[test] @@ -1367,12 +1402,14 @@ mod tests { let old = cursor.bind_request_timing(old, &session.segments[&0]); let seek = cursor.record_request_started(4, 2_000); let seek = cursor.bind_request_timing(seek, &session.segments[&4]); + cursor.record_transfer_started(old, 2_500); assert_eq!(cursor.record_request_completed(old, 2_500), HlsPlaybackCompletionOutcome::StaleRequest); + cursor.record_transfer_started(seek, 3_000); assert_eq!(cursor.record_request_completed(seek, 3_000), HlsPlaybackCompletionOutcome::Advanced); assert_eq!(cursor.first_requested_media_start_ms, Some(18_000)); assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(27_000)); let reserve = reserve(&manifest(), &cursor, &session.ready_timeline_snapshot(0, 4_000), 4_000); - assert_eq!(reserve.conservative_playback_position_ms, 19_050); - assert_eq!(reserve.guaranteed_reserve_ms, 7_950); + assert_eq!(reserve.conservative_playback_position_ms, 19_000); + assert_eq!(reserve.guaranteed_reserve_ms, 8_000); } } diff --git a/backend/hls/src/media_reserve/tests/playback.rs b/backend/hls/src/media_reserve/tests/playback.rs new file mode 100644 index 000000000..7b952743d --- /dev/null +++ b/backend/hls/src/media_reserve/tests/playback.rs @@ -0,0 +1,154 @@ +use super::*; + +#[test] +fn eof_without_transfer_evidence_does_not_start_the_clock_or_prove_exhaustion() { + let mut cursor = HlsLeasePlaybackCursor::default(); + let token = cursor.record_request_started(0, 0); + assert_eq!(cursor.record_request_completed(token, 10_000), HlsPlaybackCompletionOutcome::Advanced); + let snapshot = reserve(&manifest(), &cursor, &ready_timeline(), 100_000); + assert_eq!(cursor.first_segment_started_at_ms, None); + assert!(!snapshot.playback_position_known); + assert!(!snapshot.is_exhausted()); +} + +#[test] +fn concurrent_requests_anchor_to_the_first_actual_transfer_and_keep_its_media_start() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let first = cursor.record_request_started(0, 0); + let first = cursor.bind_request_timing(first, &session.segments[&0]); + let successor = cursor.record_request_started(1, 100); + let successor = cursor.bind_request_timing(successor, &session.segments[&1]); + cursor.record_transfer_started(successor, 2_000); + cursor.record_transfer_started(first, 2_500); + assert_eq!(cursor.first_segment_started_at_ms, Some(2_000)); + assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 3_000).conservative_playback_position_ms, 5_000); +} + +#[test] +fn first_body_chunk_starts_exactly_one_x_before_eof_and_is_not_renewed() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let request = cursor.record_request_started(0, 0); + let request = cursor.bind_request_timing(request, &session.segments[&0]); + let unknown = reserve(&manifest(), &cursor, &ready_timeline(), 2_000); + assert!(!unknown.playback_position_known); + assert!(!unknown.is_exhausted()); + assert!(cursor.record_transfer_started(request, 3_000)); + let playing = reserve(&manifest(), &cursor, &ready_timeline(), 5_000); + assert_eq!(playing.conservative_playback_position_ms, 2_000); + assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None); + assert_eq!(cursor.first_segment_completed_at_ms, None); + let retry = cursor.record_request_started(0, 6_000); + cursor.record_transfer_started(retry, 6_500); + let next = cursor.record_request_started(1, 7_000); + cursor.record_transfer_started(next, 7_500); + assert_eq!(cursor.record_request_completed(request, 10_000), HlsPlaybackCompletionOutcome::Advanced); + assert_eq!(cursor.first_segment_started_at_ms, Some(3_000)); + assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 11_000).conservative_playback_position_ms, 8_000); +} + +#[test] +fn one_hour_of_steady_supply_has_no_session_age_drift_then_exhausts_at_one_x() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let first = cursor.record_request_started(0, 0); + let first = cursor.bind_request_timing(first, &session.segments[&0]); + cursor.record_transfer_started(first, 3_000); + let mut current = manifest(); + let mut last_reserve = 0; + for tick in 0..=360 { + let media_ms = tick * 10_000; + let now_ms = 3_000 + media_ms; + let timeline = HlsReadyTimelineSnapshot { + units: (tick..tick + 6).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect(), + }; + current.first_proxy_seq = tick; + current.last_proxy_seq = tick + 5; + current.playlist_duration_ms = 60_000; + current.last_visible_media_end_ms = media_ms + 60_000; + // Completed history allows GC to remove previous metadata, while the + // original EXTINF clock and body-start anchor remain unchanged. + cursor.highest_contiguous_completed_proxy_seq = tick.checked_sub(1); + cursor.highest_contiguous_completed_media_end_ms = Some(media_ms); + let snapshot = reserve(¤t, &cursor, &timeline, now_ms); + assert_eq!(snapshot.conservative_playback_position_ms, media_ms); + assert_eq!(snapshot.guaranteed_reserve_ms, 60_000, "tick {tick}"); + last_reserve = snapshot.guaranteed_reserve_ms; + } + assert_eq!(last_reserve, 60_000); + let timeline = + HlsReadyTimelineSnapshot { units: (360..366).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() }; + for elapsed_ms in [1, 10_000, 59_999, 60_000, 70_000] { + let snapshot = reserve(¤t, &cursor, &timeline, 3_603_000 + elapsed_ms); + assert_eq!(snapshot.guaranteed_reserve_ms, 60_000_u64.saturating_sub(elapsed_ms)); + assert_eq!(snapshot.is_exhausted(), elapsed_ms >= 60_000); + } +} + +#[test] +fn fourteen_minutes_of_supply_then_a_manifest_failure_and_retry_keep_positive_reserve_live() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let first = cursor.record_request_started(0, 0); + let first = cursor.bind_request_timing(first, &session.segments[&0]); + cursor.record_transfer_started(first, 3_000); + let mut current = manifest(); + for tick in 0..=84 { + let media_ms = tick * 10_000; + let timeline = HlsReadyTimelineSnapshot { + units: (tick..tick + 3).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect(), + }; + current.first_proxy_seq = tick; + current.last_proxy_seq = tick + 2; + current.playlist_duration_ms = 30_000; + current.last_visible_media_end_ms = media_ms + 30_000; + cursor.highest_contiguous_completed_proxy_seq = tick.checked_sub(1); + cursor.highest_contiguous_completed_media_end_ms = Some(media_ms); + assert_eq!(reserve(¤t, &cursor, &timeline, 3_000 + media_ms).guaranteed_reserve_ms, 30_000); + } + let unchanged = + HlsReadyTimelineSnapshot { units: (84..87).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() }; + // One retryable manifest failure degrades the origin while the last READY + // horizon stays unchanged. The former 5% drift would already consume 42s. + for elapsed_ms in [1_000, 2_000] { + let failed = reserve(¤t, &cursor, &unchanged, 843_000 + elapsed_ms); + assert_eq!(failed.guaranteed_reserve_ms, 30_000 - elapsed_ms); + assert!(!failed.is_exhausted()); + assert!(!failed.cutover_required); + } + // The successful retry supplies one successor. Recovery neither resets + // the first-body anchor nor creates a session-age reserve penalty. + let recovered = + HlsReadyTimelineSnapshot { units: (84..88).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() }; + let healthy = evaluate_lease_reserve(HlsLeaseReserveInput { + manifest: ¤t, + cursor: &cursor, + ready_timeline: &recovered, + now_ms: 846_000, + recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000), + origin_path_degraded: false, + recovery_committed: true, + }); + assert_eq!(healthy.conservative_playback_position_ms, 843_000); + assert_eq!(healthy.guaranteed_reserve_ms, 37_000); + assert!(!healthy.cutover_required); + assert_eq!(cursor.first_segment_started_at_ms, Some(3_000)); +} + +#[test] +fn seek_discards_the_start_anchor_and_rejects_old_first_chunks() { + let session = variable_duration_session(); + let mut cursor = HlsLeasePlaybackCursor::default(); + let old = cursor.record_request_started(0, 0); + let old = cursor.bind_request_timing(old, &session.segments[&0]); + cursor.record_transfer_started(old, 1_000); + let seek = cursor.record_request_started(4, 2_000); + let seek = cursor.bind_request_timing(seek, &session.segments[&4]); + assert_eq!(cursor.first_segment_started_at_ms, None); + assert!(!cursor.record_transfer_started(old, 2_500)); + assert!(!reserve(&manifest(), &cursor, &ready_timeline(), 2_500).playback_position_known); + cursor.record_transfer_started(seek, 3_000); + assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 5_000).conservative_playback_position_ms, 20_000); + assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None); +} diff --git a/backend/hls/src/post_refresh_availability.rs b/backend/hls/src/post_refresh_availability.rs index 0aaa59b47..6fcc7d55e 100644 --- a/backend/hls/src/post_refresh_availability.rs +++ b/backend/hls/src/post_refresh_availability.rs @@ -7,7 +7,7 @@ use super::{ availability_reevaluation::{HlsAvailabilityReevaluationOwnership, HlsAvailabilityReevaluationRegistration}, hls_ctx::HlsCtx, media_reserve::HlsLeaseReserveSnapshot, - recovery_timing::{HlsLeaseCutoverTiming, HlsTerminalCommitAcquisitionBudgetMs}, + recovery_timing::HlsLeaseCutoverTiming, refresh::HlsPostRefreshAvailabilityAction, HlsAccessLeaseId, HlsSessionHandle, ProxySessionId, }; @@ -137,17 +137,13 @@ pub(super) fn live_reserve_deadline( reserve: HlsLeaseReserveSnapshot, cutover_timing: HlsLeaseCutoverTiming, ) -> Option { - let trigger_reserve_ms = reserve - .transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); - let next_reevaluation_at_ms = - now_ms.saturating_add(reserve.guaranteed_reserve_ms.saturating_sub(trigger_reserve_ms)); - let latest_safe_terminal_commit_at_ms = cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - let latest_wake_at_ms = latest_safe_terminal_commit_at_ms.checked_sub(1)?; + if !reserve.playback_position_known || reserve.guaranteed_reserve_ms == 0 { + return None; + } + let exhaustion_at_ms = cutover_timing.lease_exhaustion_at.as_millis_since_epoch(); Some(HlsLiveReserveDeadline { - next_reevaluation_at_ms: next_reevaluation_at_ms.min(latest_wake_at_ms), - latest_safe_terminal_commit_at_ms, + next_reevaluation_at_ms: exhaustion_at_ms.max(now_ms.saturating_add(1)), + latest_safe_terminal_commit_at_ms: exhaustion_at_ms, }) } @@ -157,7 +153,7 @@ pub(super) async fn evaluate_active_terminal_leases_for_reevaluation( proxy_session_id: &ProxySessionId, now_ms: u64, ) -> HlsPostRefreshTerminalEvaluation { - let leases = ctx.hls_proxy.active_live_playback_snapshots_for_session(proxy_session_id, now_ms).await; + let leases = ctx.hls_proxy.terminal_playback_snapshots_for_session(proxy_session_id, now_ms).await; if leases.is_empty() { return HlsPostRefreshTerminalEvaluation::NoLiveLease; } @@ -233,7 +229,7 @@ pub(super) async fn evaluate_owner_failure_fallback( proxy_session_id: &ProxySessionId, now_ms: u64, ) -> HlsPostRefreshFallbackAggregate { - let leases = ctx.hls_proxy.active_live_playback_snapshots_for_session(proxy_session_id, now_ms).await; + let leases = ctx.hls_proxy.terminal_playback_snapshots_for_session(proxy_session_id, now_ms).await; let mut aggregate = HlsPostRefreshFallbackAggregate::default(); for lease in &leases { let HlsDetailedTerminalResolution { resolution, live_reserve_deadline } = diff --git a/backend/hls/src/recovery_timing.rs b/backend/hls/src/recovery_timing.rs index f7ef7f99b..16d311bb2 100644 --- a/backend/hls/src/recovery_timing.rs +++ b/backend/hls/src/recovery_timing.rs @@ -7,6 +7,11 @@ const HLS_ACCEPTANCE_COMMIT_ETA_MS: u64 = 100; const HLS_ACCEPTANCE_SCHEDULING_ETA_MS: u64 = 100; const HLS_MANIFEST_REQUEST_FALLBACK_ETA_MS: u64 = 3_000; +/// Decision budget derived from the session's publication cadence, not an HTTP timeout. +pub const fn recovery_budget_ceiling_ms(target_duration_ms: u64) -> u64 { + target_duration_ms.saturating_add(target_duration_ms.div_ceil(2)) +} + /// Converts a hard per-operation limit into a conservative fallback estimate /// without treating unusually large protection timeouts as expected latency. pub const fn bounded_manifest_request_eta_ms(operation_timeout_ms: u64) -> u64 { @@ -76,7 +81,6 @@ impl HlsEstimatedRecoveryCompletionAtMs { } impl HlsLeaseExhaustionAtMs { - #[cfg(any(test, feature = "test-support"))] pub const fn as_millis_since_epoch(self) -> u64 { self.0 } } @@ -99,9 +103,8 @@ impl HlsTerminalCommitAcquisitionBudgetMs { pub const fn as_millis(self) -> u64 { self.0 } } -/// Lease-local phase of terminal acquisition. `AcquisitionOpen` is not an -/// admission or origin-progress transition; it only permits the bounded final -/// commit to begin before the unchanged product transition margin. +/// Lease-local preparation pressure. Neither `AcquisitionOpen` nor the old +/// transition boundary proves exhaustion or authorizes CSV publication. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HlsTerminalCommitWindow { NotDue, @@ -401,6 +404,11 @@ impl HlsAcceptanceEpisodeTiming { calculated_eta_ms.max(input.observed_latency.p95.map_or(0, HlsRecoveryEtaMs::as_millis)); let initial_recovery_eta_ms = calculated_eta_ms.max(input.observed_latency.p99.map_or(trigger_budget_ms, HlsRecoveryEtaMs::as_millis)); + let ceiling_ms = recovery_budget_ceiling_ms(input.target_duration_ms); + log::debug!("HLS recovery decision budget: session_target_duration_ms={} raw_trigger_ms={trigger_budget_ms} raw_eta_ms={initial_recovery_eta_ms} ceiling_ms={ceiling_ms} effective_trigger_ms={} effective_eta_ms={}", + input.target_duration_ms, trigger_budget_ms.min(ceiling_ms), initial_recovery_eta_ms.min(ceiling_ms)); + let trigger_budget_ms = trigger_budget_ms.min(ceiling_ms); + let initial_recovery_eta_ms = initial_recovery_eta_ms.min(ceiling_ms); let deadline_budget_ms = trigger_budget_ms.max(initial_recovery_eta_ms); Self { manifest_operation_timeout: input.policy.manifest_operation_timeout, @@ -438,11 +446,9 @@ impl HlsAcceptanceEpisodeTiming { media_eta, self.scheduling_eta, )); - if workload == self.initial_workload { - workload_eta.max(self.initial_recovery_eta) - } else { - workload_eta - } + let eta = + if workload == self.initial_workload { workload_eta.max(self.initial_recovery_eta) } else { workload_eta }; + HlsRecoveryEtaMs::from_millis(eta.as_millis().min(recovery_budget_ceiling_ms(self.target_duration_ms))) } pub fn estimated_completion_at( @@ -451,7 +457,9 @@ impl HlsAcceptanceEpisodeTiming { workload: HlsRecoveryWorkload, ) -> HlsEstimatedRecoveryCompletionAtMs { HlsEstimatedRecoveryCompletionAtMs::from_millis_since_epoch( - now_ms.saturating_add(self.remaining_eta(workload).as_millis()), + now_ms + .saturating_add(self.remaining_eta(workload).as_millis()) + .min(self.acceptance_deadline.as_millis_since_epoch()), ) } } @@ -604,6 +612,51 @@ mod tests { }) } + #[test] + fn all_decision_budgets_are_capped_after_latency_and_workload_without_changing_timeouts() { + for target_duration_ms in [1, 999, 10_000, 30_001, u64::MAX] { + let workload = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(); + let input = HlsAcceptanceEpisodeTimingInput { + started_at_ms: 1_000, + burst_plan: HlsManifestRecoveryBurstLevel::Beast.plan(), + target_duration_ms, + transition_margin: HlsTransitionMarginMs::from_millis(target_duration_ms), + workload, + observed_latency: HlsObservedRecoveryLatency { + p95: Some(HlsRecoveryEtaMs::from_millis(u64::MAX)), + p99: Some(HlsRecoveryEtaMs::from_millis(u64::MAX)), + }, + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + policy: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(1_000_000), + HlsOperationTimeoutMs::from_millis(2_000_000), + HlsRecoveryEtaMs::from_millis(1_000_000), + HlsRecoveryEtaMs::from_millis(2_000_000), + ), + }; + let timing = HlsAcceptanceEpisodeTiming::from_input(&input); + let ceiling = recovery_budget_ceiling_ms(target_duration_ms); + assert_eq!(timing.trigger_budget.as_millis(), ceiling); + assert_eq!(timing.initial_recovery_eta.as_millis(), ceiling); + for remaining in [workload, workload.after_full_burst(), HlsRecoveryWorkload::clear_fetch()] { + assert!(timing.remaining_eta(remaining).as_millis() <= ceiling); + for now_ms in [1_000, 2_000, 50_000] { + assert!( + timing.estimated_completion_at(now_ms, remaining).as_millis_since_epoch() + <= timing.acceptance_deadline.as_millis_since_epoch() + ); + } + } + assert_eq!(timing.manifest_operation_timeout, input.policy.manifest_operation_timeout); + assert_eq!(timing.media_operation_timeout, input.policy.media_operation_timeout); + } + let small = timing(HlsRecoveryWorkload::clear_fetch()); + assert!(small.trigger_budget.as_millis() < 15_000); + assert_eq!(recovery_budget_ceiling_ms(10_000), 15_000); + assert_eq!(recovery_budget_ceiling_ms(999), 1_499); + } + #[test] fn hls_recovery_timing_workload_classes_have_distinct_expected_etas() { let staged = timing(HlsRecoveryWorkload { @@ -720,7 +773,7 @@ mod tests { assert_eq!(frozen.observed_latency.p99.map(HlsRecoveryEtaMs::as_millis), Some(14_000)); assert_eq!(frozen.remaining_eta(frozen.initial_workload), frozen.initial_recovery_eta); - assert_ne!(frozen.acceptance_deadline, later.acceptance_deadline); + assert_eq!(frozen.acceptance_deadline, later.acceptance_deadline); assert!(frozen.initial_recovery_eta.as_millis() < frozen.media_operation_timeout.as_millis()); assert_eq!(bounded_manifest_request_eta_ms(120_000), 3_000); assert_eq!(bounded_manifest_request_eta_ms(750), 750); @@ -812,8 +865,8 @@ mod tests { }); assert_eq!(timing.acceptance_deadline.as_millis_since_epoch(), u64::MAX); - assert_eq!(timing.trigger_budget.as_millis(), u64::MAX); - assert_eq!(timing.initial_recovery_eta.as_millis(), u64::MAX); + assert_eq!(timing.trigger_budget.as_millis(), 15_000); + assert_eq!(timing.initial_recovery_eta.as_millis(), 15_000); } #[test] diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index 8c408f3d0..fbe16905e 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -28,9 +28,8 @@ use super::{ HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEncryptionReadiness, HlsRecoveryEtaMs, HlsRecoveryMapWorkload, HlsRecoveryMediumReadiness, HlsRecoveryObjectReadiness, HlsRecoverySegmentWorkload, - HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope, - HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, - HlsTransitionMarginMs, + HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope, HlsTerminalCommitWindow, + HlsTerminalMediaPreparationState, HlsTransitionMarginMs, }, runtime_custom_tail::HlsRuntimeCustomTailReason, session_store::HlsSessionIncarnation, diff --git a/backend/hls/src/refresh/tests/acceptance.rs b/backend/hls/src/refresh/tests/acceptance.rs index 30f9f1f1b..10c31e14f 100644 --- a/backend/hls/src/refresh/tests/acceptance.rs +++ b/backend/hls/src/refresh/tests/acceptance.rs @@ -678,7 +678,6 @@ pub(in crate::refresh::tests) async fn assert_critical_handoff_evidence( cursor: &live_lease.playback_cursor, ready_timeline: &ready_timeline, now_ms, - playback_rate_guard_milli: super::super::super::HLS_PLAYBACK_RATE_GUARD_MILLI, recovery_trigger_budget: super::super::super::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0), origin_path_degraded: true, recovery_committed: false, diff --git a/backend/hls/src/refresh/tests/retry_policy.rs b/backend/hls/src/refresh/tests/retry_policy.rs index 61b724d69..7d8705f45 100644 --- a/backend/hls/src/refresh/tests/retry_policy.rs +++ b/backend/hls/src/refresh/tests/retry_policy.rs @@ -14,10 +14,10 @@ use super::{ HlsManifestFetchFailureSignal, HlsManifestHttpResponseEvidence, HlsManifestProgress, HlsManifestRecoveryUnavailableReason, HlsManifestRejectLogReason, HlsManifestSequenceRelation, HlsOriginManifestFetchContext, HlsOriginManifestFetchRequest, HlsOriginRefreshTriggerOutcome, - HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsTerminalCommitAcquisitionBudgetMs, - HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalTailPreparationRequest, - HlsTransitionMarginMs, LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, - OriginRefreshRequest, OriginRefreshState, RetryPolicy, + HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsTerminalCommitPayload, + HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs, + LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, OriginRefreshRequest, OriginRefreshState, + RetryPolicy, }; use crate::{ media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot}, @@ -927,13 +927,12 @@ async fn hls_cutover_policy_advanced_commit_supersedes_preparation_and_keeps_lea .is_committed()); let prepared_progress_generation = prepare_failed_acceptance_episode(&session).await; let transition_margin = HlsTransitionMarginMs::from_millis(4_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + let guaranteed_reserve_ms = 0; let reserve = HlsLeaseReserveSnapshot { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: 8_000_u64.saturating_add(guaranteed_reserve_ms), conservative_playback_position_ms: 8_000, + playback_position_known: true, guaranteed_reserve_ms, initial_hidden_ready_duration_ms: 0, transition_margin, diff --git a/backend/hls/src/resource_fetch.rs b/backend/hls/src/resource_fetch.rs index b0f6fe536..cac727a2a 100644 --- a/backend/hls/src/resource_fetch.rs +++ b/backend/hls/src/resource_fetch.rs @@ -641,7 +641,9 @@ async fn fetch_hls_origin_resource_response_with_timeout( let response = transport::send(client, request, timeout, &mut deadline).await?; if !clients.use_manual_redirects || !response.status().is_redirection() { let response = classify_hls_origin_resource_response(response, target.byte_range_expectation)?; - return Ok((response, deadline.expect("transport returned a connection deadline"))); + let deadline = deadline + .ok_or_else(|| HlsOriginResourceFetchError::Transport("HLS connection deadline unavailable".into()))?; + return Ok((response, deadline)); } if remaining_redirects == 0 { return Err(HlsOriginResourceFetchError::Redirect); diff --git a/backend/hls/src/resource_fetch/transport.rs b/backend/hls/src/resource_fetch/transport.rs index 283e60b40..6e549e35d 100644 --- a/backend/hls/src/resource_fetch/transport.rs +++ b/backend/hls/src/resource_fetch/transport.rs @@ -34,7 +34,8 @@ pub(super) async fn send( } } } - let deadline = deadline.expect("successful connection establishes the deadline"); + let deadline = + deadline.ok_or_else(|| HlsOriginResourceFetchError::Transport("HLS connection deadline unavailable".into()))?; tokio::time::timeout_at(deadline.deadline(), response) .await .map_err(|_| HlsOriginResourceFetchError::Timeout)? diff --git a/backend/hls/src/response/activity.rs b/backend/hls/src/response/activity.rs index 8a79a1b3d..446523069 100644 --- a/backend/hls/src/response/activity.rs +++ b/backend/hls/src/response/activity.rs @@ -33,6 +33,7 @@ impl HlsMediaActivityMarker { &marker.lease_id, &marker.proxy_session_id, marker.lease_identity, + marker.completed_segment, current_time_millis(), ) .await; diff --git a/backend/hls/src/response/cache.rs b/backend/hls/src/response/cache.rs index 6f5e20ef0..e657a2d8f 100644 --- a/backend/hls/src/response/cache.rs +++ b/backend/hls/src/response/cache.rs @@ -375,7 +375,7 @@ where mark_response_as_uncompressed(&mut response); if object.is_media { if let Some(marker) = &context.media_activity_marker { - response = marker.confirm_media_response(response); + response = cursor_marker.as_ref().unwrap_or(marker).confirm_media_response(response); } } Ok(response) diff --git a/backend/hls/src/response/revision.rs b/backend/hls/src/response/revision.rs index 707ae243f..1a6279e36 100644 --- a/backend/hls/src/response/revision.rs +++ b/backend/hls/src/response/revision.rs @@ -130,7 +130,7 @@ async fn prepare_hls_revision_response( let stream = RevisionResponseStream { inner: super::super::progressive_startup::revision_body(revision, lifetime, body_range.start, body_range.end), qos_meter: Arc::clone(&context.qos_meter), - completion, + completion: completion.clone(), observation, expected_len: body_range.length, bytes_sent: 0, @@ -142,7 +142,7 @@ async fn prepare_hls_revision_response( body_range.apply_headers(&mut response, metadata.as_ref(), context.cache_duration_seconds); mark_response_as_uncompressed(&mut response); context.metrics.record_cache_hit(); - HlsResourceServeOutcome::Ready(marker.confirm_media_response(response)) + HlsResourceServeOutcome::Ready(completion.as_ref().unwrap_or(marker).confirm_media_response(response)) } fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::Instant { diff --git a/backend/hls/src/response/tests/lifecycle.rs b/backend/hls/src/response/tests/lifecycle.rs index 11951381e..5afaaf2ff 100644 --- a/backend/hls/src/response/tests/lifecycle.rs +++ b/backend/hls/src/response/tests/lifecycle.rs @@ -1,5 +1,28 @@ use super::*; +#[tokio::test] +async fn empty_body_chunks_do_not_start_playback_but_the_first_data_chunk_does() { + let fixture = CachedSegmentFixture::new(vec![9; 512]).await; + let marker = fixture + .context + .media_activity_marker + .clone() + .unwrap() + .for_segment_request(fixture.segment_file.proxy_seq, current_time_millis()) + .await + .unwrap(); + let chunks = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::new()), Ok(Bytes::from_static(b"media"))]); + let response = marker.confirm_media_response(axum::http::Response::new(axum::body::Body::from_stream(chunks))); + let mut body = response.into_body(); + assert_eq!(body.frame().await.unwrap().unwrap().into_data().unwrap(), Bytes::new()); + assert_eq!(fixture.cursor().await.first_segment_started_at_ms, None); + assert_eq!(body.frame().await.unwrap().unwrap().into_data().unwrap(), Bytes::from_static(b"media")); + let cursor = fixture.cursor().await; + assert!(cursor.first_segment_started_at_ms.is_some()); + assert_eq!(cursor.first_segment_completed_at_ms, None); + assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None); +} + #[tokio::test] async fn progressive_http_drop_errors_without_content_length_or_cursor_completion() -> std::io::Result<()> { let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await; @@ -105,6 +128,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() { assert_eq!(requested.first_requested_proxy_seq, Some(12)); assert_eq!(requested.last_requested_proxy_seq, Some(12)); assert_eq!(requested.highest_contiguous_completed_proxy_seq, None); + assert_eq!(requested.first_segment_started_at_ms, None); assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); let mut body = response.into_body(); @@ -126,6 +150,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() { assert_eq!(dropped.last_requested_proxy_seq, Some(12)); assert_eq!(dropped.highest_contiguous_completed_proxy_seq, None); assert_eq!(dropped.first_segment_completed_at_ms, None); + assert!(dropped.first_segment_started_at_ms.is_some()); assert_eq!(fixture.meter.bytes_total(), u64::try_from(first_chunk.len()).expect("first chunk size")); assert_eq!(fixture.access.active_readers(), 0); } diff --git a/backend/hls/src/response/tests/revision_recovery.rs b/backend/hls/src/response/tests/revision_recovery.rs index 7c874f239..e8318daed 100644 --- a/backend/hls/src/response/tests/revision_recovery.rs +++ b/backend/hls/src/response/tests/revision_recovery.rs @@ -774,7 +774,7 @@ async fn origin_generation_and_mapping_changes_never_install_retired_retry_bytes #[tokio::test] async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> std::io::Result<()> { let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await; - let failed = failed_optimistic_revision(&fixture, 6).await?; + let _failed = failed_optimistic_revision(&fixture, 6).await?; install_completed_retry(&fixture).await?; let mut leases = fixture.manager.access_leases().write().await; let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard"); @@ -788,7 +788,7 @@ async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> HlsRevisionRecoveryOutcome::Unchanged ); assert!(leases.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, expires + 1).is_none()); - assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed)); + assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), None); Ok(()) } diff --git a/backend/hls/src/terminal_commit.rs b/backend/hls/src/terminal_commit.rs index 6b07edb18..7dfef779f 100644 --- a/backend/hls/src/terminal_commit.rs +++ b/backend/hls/src/terminal_commit.rs @@ -329,10 +329,11 @@ impl HlsTerminalCommitCommand { } fn has_independent_commit_deadline(&self) -> bool { - matches!( - &self.decision, - HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) - ) + !self.preparation.trigger.is_runtime_policy() + || matches!( + &self.decision, + HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) + ) } fn may_replace(&self, current: &Self) -> bool { @@ -1357,6 +1358,7 @@ mod tests { availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, guaranteed_media_horizon_ms: deadline_ms.saturating_add(100), conservative_playback_position_ms: 0, + playback_position_known: true, guaranteed_reserve_ms: deadline_ms.saturating_add(100), initial_hidden_ready_duration_ms: 0, transition_margin, @@ -1365,7 +1367,9 @@ mod tests { cutover_required: true, }; let preparation = HlsTerminalTailPreparation { - trigger: super::super::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve, + trigger: super::super::runtime_custom_tail::HlsFiniteTailTrigger::RuntimePolicy( + HlsRuntimeCustomTailReason::LowPriorityPreempted, + ), runtime_policy_revocation: None, lease_issued_at_ms: 1, decision_generation: 1, @@ -1428,6 +1432,29 @@ mod tests { } } + #[test] + fn availability_submission_gets_one_fixed_cas_budget_after_the_old_splice_deadline() { + let coordinator = HlsTerminalCommitRetryCoordinator::default(); + let mut command = retry_command("availability-budget", 1, 1, 1); + command.preparation.trigger = crate::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve; + command.preparation.reserve.guaranteed_reserve_ms = 0; + let key = command.key.clone(); + let first_deadline = 100 + terminal_commit_retry_schedule_budget_ms(); + assert!(matches!(coordinator.submit(command, 100), HlsTerminalCommitSubmissionDecision::Attempt { .. })); + assert_eq!( + coordinator.lock_state().owners.get(&key).map(|owner| owner.effective_deadline_ms), + Some(first_deadline) + ); + let mut replay = retry_command("availability-budget", 1_000_000, 2, 1); + replay.preparation.trigger = crate::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve; + replay.preparation.reserve.guaranteed_reserve_ms = 0; + let _ = coordinator.submit(replay, 200); + assert_eq!( + coordinator.lock_state().owners.get(&key).map(|owner| owner.effective_deadline_ms), + Some(first_deadline) + ); + } + fn scheduled_worker_token( decision: &HlsTerminalCommitRetryScheduleDecision, ) -> super::HlsTerminalCommitWorkerToken { diff --git a/backend/hls/src/terminal_tail.rs b/backend/hls/src/terminal_tail.rs index 876d3e240..45a1f82f5 100644 --- a/backend/hls/src/terminal_tail.rs +++ b/backend/hls/src/terminal_tail.rs @@ -106,6 +106,7 @@ struct HlsTerminalBaseSegmentProbe { media_ready: bool, required_map_ready: bool, cache_key: Option, + revision_metadata: Option, origin_epoch: Option, key_evidence: HlsTerminalBaseKeyEvidence, } diff --git a/backend/hls/src/terminal_tail/evidence.rs b/backend/hls/src/terminal_tail/evidence.rs index dc736b2d6..e09f722a9 100644 --- a/backend/hls/src/terminal_tail/evidence.rs +++ b/backend/hls/src/terminal_tail/evidence.rs @@ -62,7 +62,7 @@ impl HlsTerminalBaseEvidence { /// Makes the intended guard lifetime explicit at the endpoint commit boundary. #[cfg(any(test, feature = "test-support"))] - pub fn release(self) { drop(self.read_protection) } + pub fn release(self) { drop(self.read_protection); } /// Transfers the READY-object reader pins to an autonomous terminal commit. /// @@ -183,7 +183,16 @@ fn snapshot_terminal_base_probes( .iter() .map(|segment| { let entry = session.segments.get(&segment.proxy_seq); - let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. })); + let revision_metadata = manifest + .startup_revisions + .as_ref() + .and_then(|publication| publication.revisions.get(&segment.proxy_seq)) + .and_then(|bound| match bound.revision().state() { + crate::SegmentRevisionState::Complete(metadata) => Some(metadata), + _ => None, + }); + let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. })) + && (manifest.startup_revisions.is_none() || revision_metadata.is_some()); let required_map_ready = entry.is_some_and(|entry| { entry.map_ref.is_none_or(|map_id| { session @@ -215,6 +224,7 @@ fn snapshot_terminal_base_probes( media_ready, required_map_ready, cache_key, + revision_metadata, origin_epoch: media_ready.then(|| entry.map(|entry| entry.origin_key.origin_epoch)).flatten(), key_evidence, } @@ -223,6 +233,19 @@ fn snapshot_terminal_base_probes( HlsTerminalBaseEvidencePreparation { probes, read_protection: HlsTerminalBaseReadProtection { accesses } } } +async fn terminal_probe_metadata( + cache: &HlsSegmentCache, + probe: &HlsTerminalBaseSegmentProbe, +) -> Option { + if let Some(metadata) = probe.revision_metadata.as_ref() { + return Some(metadata.clone()); + } + match probe.cache_key.as_ref() { + Some(key) => cache.metadata(key).await.ok().flatten(), + None => None, + } +} + async fn resolve_terminal_base_probes( segment_cache: &HlsSegmentCache, manifest: &HlsLeaseManifestSnapshot, @@ -235,11 +258,7 @@ async fn resolve_terminal_base_probes( let mut last_track_preprobe_resolution = None; let mut track_base = None; for probe in probes { - let metadata = if let Some(cache_key) = probe.cache_key.as_ref() { - segment_cache.metadata(cache_key).await.ok().flatten() - } else { - None - }; + let metadata = terminal_probe_metadata(segment_cache, &probe).await; let protectable = metadata.is_some(); let (required_key_ready, track_encryption, key_failure) = match probe.key_evidence { HlsTerminalBaseKeyEvidence::Clear => (true, Some(HlsTerminalTrackEncryption::Clear), None), diff --git a/docs/src/configuration/hls-cache-state-machine.md b/docs/src/configuration/hls-cache-state-machine.md index 208e9891f..6c3cfd422 100644 --- a/docs/src/configuration/hls-cache-state-machine.md +++ b/docs/src/configuration/hls-cache-state-machine.md @@ -220,7 +220,7 @@ The manifest commit policy tracks multiple signals: | :--- | :--- | :--- | | `PublicationLate` | No publication progress has been observed for `1.5 * TARGETDURATION`. | Lateness evidence only. By itself it neither switches host nor closes admission nor cuts over a lease. | | `RecoveryRequired` | Degraded origin evidence and lease reserve/ETA show that recovery work must start. | Opens one generation-bound acceptance episode and temporarily closes new admission while existing leases use their remaining READY reserve. | -| `Critical` | At least one live lease has reached its reserve/transition boundary without committed recovery. | Evaluates that lease's safe cross-host handoff or prepared terminal decision before its media exhaustion deadline. | +| `Critical` | At least one live lease has reached its reserve/transition boundary without committed recovery. | Prepares that lease's safe cross-host handoff or terminal media. Availability CSV publication still requires known reserve exhaustion. | | `NoAdmission` | Operator shorthand for the closed startup-admission gate; it is not a separate stored origin enum. | New leases are not admitted from insufficient READY startup media. Existing leases are evaluated independently and are not terminalized as a group. | An acceptance episode owns an immutable candidate plan, timing budget, workload binding, generation, and deadline. The @@ -250,7 +250,7 @@ A segment request validates the access lease and then looks up the segment objec | Known but not ready | Wait briefly or return `503` with `Retry-After`. | | Missing from session timeline | Return an unavailable/expired response depending on context. | | Retryable or hard origin failure | Update origin-path evidence and bounded retry state; keep serving lease-specific READY reserve when safe. | -| Lease reserve reaches the transition boundary | Commit a generation-bound prepared terminal decision only after the complete acceptance budget can no longer recover before the safe deadline. | +| Lease reserve reaches the transition boundary | Prepare and re-evaluate early; publish availability CSV only at known reserve exhaustion while the origin is still degraded. | For a warm canonical manifest request, terminal `channel_unavailable` is not a redirect. The same manifest URL returns an immutable HTTP `200` media playlist containing a safe live suffix, a discontinuity, finite lease/generation-bound @@ -304,7 +304,7 @@ the old location. | Target duration fallback | 15 seconds | Used before an origin target duration is known. | | Publication lateness | `1.5 * target_duration` | Observation signal only; never a cutover or terminal deadline. | | Recovery trigger | Lease READY reserve minus full acceptance ETA and transition margin | Starts recovery while the configured burst can still complete. | -| Terminal cutover | Lease-specific safe splice deadline | Bounds a live-tail splice; a late finite Custom Response uses a separate bounded commit budget. | +| Terminal cutover | Known lease media exhaustion | Starts one bounded terminal CAS/retry budget; the last committed live segment remains before CSV. | | HLS cache GC interval | 30 seconds | Periodic cleanup cadence for cache objects and stale runtime state. | | Temporary file retention | 30 seconds | Retention window for temporary HLS cache files. | | Failed segment retention | 10 seconds | Short retention for failed segment state. | diff --git a/docs/src/configuration/shared-hls-runtime-flow.md b/docs/src/configuration/shared-hls-runtime-flow.md index f954a1308..064a13f9e 100644 --- a/docs/src/configuration/shared-hls-runtime-flow.md +++ b/docs/src/configuration/shared-hls-runtime-flow.md @@ -190,7 +190,8 @@ origin epoch with a discontinuity when required. `PublicationLate` begins at `1.5 * TARGETDURATION`, but it is only an observation that the origin is publishing late. The runtime starts recovery and closes startup admission only when lease-specific READY reserve, full-burst recovery ETA, and -transition margin require it. It cuts over only at the lease-specific safe deadline. Manifest request counts never +transition margin require it. For availability failures, CSV is published only after the lease's known media reserve is +completely consumed and the origin remains degraded. Manifest request counts never terminalize a lease, and initial strip contributes only the READY media actually available to that lease and its measured start distance. @@ -216,16 +217,36 @@ offset_pcr(x) = offset_90khz(x) * 300 ``` The measured terminal asset duration determines the timestamp stride and the `#EXTINF` value. The target duration is -retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments. Both the -live-tail splice and the late Custom Response use this same prepared bundle. +retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments. +The terminal tail uses this prepared bundle while preserving the lease's live base. ## Warm terminal-tail response -If recovery cannot commit before one lease's safe cutover deadline, Tuliprox evaluates the prepared bundle against the -lease's safe READY live suffix. MPEG-TS terminal media is permitted only for a compatible TS container, inactive -`EXT-X-MAP`, compatible track layout, and a key state that can safely reset to clear media. A required encryption reset is -announced with `#EXT-X-KEY:METHOD=NONE`; an fMP4 MAP or unsupported key transition fails closed instead of creating a TS -splice. +A live lease starts its playback clock at the first nonempty body chunk of a qualified whole-segment GET (including a +full `206 bytes=0-` response). Its estimated position advances at exactly 1x from that segment's EXTINF media start. EOF +separately proves complete delivery; an aborted or in-flight body does not become completed playback history. Manifest +polls, prefetch and ordinary successor requests do not restart the clock. A valid seek starts a new playback section. +Without a transfer/media-time anchor the position is unknown, which does not prove exhaustion. + +Contiguous READY media, completed delivery history and required MAP/key readiness determine the available horizon. This +media reserve is not capped. Recovery trigger and ETA budgets are separately limited to +`ceil(1.5 * session TARGETDURATION)` after workload and P95/P99 estimates. HTTP operation timeouts stay independent. +An acceptance episode retains its original deadline; subsequent ETA evaluations cannot move its completion estimate beyond it. + +Recovery and bundle preparation may start early, including inside the transition margin and acquisition window. Positive +reserve never authorizes availability CSV publication. Immediate policy cutovers (for example preemption, maintenance or +account blocking) retain their separate rules. The independent terminal observer wakes at reserve exhaustion or on new +evidence, including while an origin retry is in flight and without another player poll. + +At proven exhaustion with a still-degraded origin, Tuliprox evaluates the prepared bundle against the last client +manifest committed to that lease. The safe suffix must include that exact manifest's last live segment, with its URI, +EXTINF and bound revision unchanged. Its metadata, dependencies and revision bytes remain protected during the live +lease; terminal publication atomically transfers protection to the immutable tail. A successor publication or lease end +releases the previous mandatory anchor protection. + +MPEG-TS terminal media still requires a compatible TS container, inactive `EXT-X-MAP`, compatible track layout, and a +safe key transition. A required encryption reset is announced with `#EXT-X-KEY:METHOD=NONE`. Missing bases, invalid +assets and incompatible media retain their explicit failure reason. The canonical manifest URL then returns HTTP `200` without `Location`. Its immutable body contains the safe live tail, the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment references, and `#EXT-X-ENDLIST`: @@ -234,24 +255,11 @@ the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment r /hls/shared/live///terminal//.ts ``` -If the safe splice deadline has already elapsed, a valid lease with a previously published manifest instead commits a -finite Custom Response without a live suffix. Its canonical manifest returns HTTP `200` with no redirect, twelve -prepared `channel_unavailable` segments under `/cvs/hls///.ts`, a clear-key -reset, a discontinuity and `#EXT-X-ENDLIST`. The target duration is retained; media sequence starts after the highest -segment number previously published to that lease, and discontinuity sequence does not decrease. No live cache base, -origin request or timestamp rewrite is needed to serve these custom segments. Playback may stall before this late -warning becomes visible: the fallback does not promise a seamless transition after the deadline. - -The late commit has one bounded retry budget independent of the expired splice deadline. Repeated playlist requests -cannot extend it. Lease incarnation, publication/cursor generation, origin recovery and configured asset revision -are revalidated at commit. The frozen manifest and media remain unchanged on replay and asset reload; access still -requires the valid lease and existing user authorization. Successful authorized GET, HEAD and range requests for the -late custom segments activate or renew that exact lease and refresh session activity using the configured -`session_idle_timeout`; playback does not depend on further manifest polls after `#EXT-X-ENDLIST`. Invalid routes, -unsatisfiable ranges and denied authorization do not renew the lease, and an expired lease cannot be reactivated. -Disabled, missing or incompatible custom media can retain an error response. Preparation failures retain their typed -compatibility reason, including `TargetDurationExceeded` with the measured asset and playlist target durations. -The original compatible live-tail splice continues to apply before its deadline. +The former pre-exhaustion splice deadline does not authorize an earlier commit or a standalone CSV fallback. A published +live lease retains its last live segment even if that deadline has elapsed. The first eligible terminal submission +starts one bounded CAS/retry budget; repeated playlist requests cannot renew it. Lease, manifest/cursor generations, +origin progress, asset identity and bound revisions are revalidated at commit. Recovery or a newer client manifest +supersedes a stale decision. A successful terminal publication stays immutable on replay and asset reload. Only the lease- and generation-bound terminal route serves the live-tail splice's prepared terminal bytes and finite full/range responses. Existing diff --git a/docs/src/configuration/shared-hls-troubleshooting.md b/docs/src/configuration/shared-hls-troubleshooting.md index 9aa3372d9..cf1a5f54c 100644 --- a/docs/src/configuration/shared-hls-troubleshooting.md +++ b/docs/src/configuration/shared-hls-troubleshooting.md @@ -88,13 +88,19 @@ Fix: ## Symptom: custom video `channel_unavailable` -For the warm Shared HLS path, this is a lease-local terminal decision rather than a failure-counter threshold. The -canonical manifest stays on the same URL and returns an HTTP `200` finite playlist when a compatible terminal tail is -ready. Before the splice deadline, an incompatible MAP/key/container/track state can fail closed with `503`. -After the deadline, a valid published lease can receive a finite Custom Response containing only prepared -`channel_unavailable` media, even when its live cache suffix is gone. The playlist uses `/cvs/hls/` URLs and does not -attempt a late TS splice. The player may stall before displaying it. Disabled, missing or invalid custom media, -expired authorization, and exhausted commit capacity can still prevent a usable response. +For an established live lease, an availability failure may publish `channel_unavailable` only after known reserve is +completely consumed and the origin remains degraded. Preparation and recovery may begin earlier. The canonical manifest +returns HTTP `200` with the last live segment of the lease's previous client manifest, the CSV tail and `ENDLIST`. That +live URI and its bound bytes remain available through GC. + +There is no CSV-only fallback after the old splice deadline. Missing transfer evidence does not prove reserve +exhaustion. Disabled, missing or invalid custom media, an unavailable base, incompatible MAP/key/container/track state, +expired authorization or exhausted commit capacity retain a concrete error response. + +The playback clock begins at the first nonempty qualified segment body chunk and advances at 1x; EOF separately proves +completion. Session age alone adds no reserve penalty. Recovery/ETA decisions are capped at 1.5 session target +durations, independently of media reserve and HTTP timeouts. Inspect `HLS terminal reserve decision` and +`HLS recovery decision budget` for anchor, position, reserve, snapshot generation and raw/effective budgets. Possible causes: @@ -102,7 +108,7 @@ Possible causes: | :--- | :--- | | Origin progress | Repeated valid HTTP `200` manifests without host-local media progress, or retryable/hard fetch evidence. | | Manifest acceptance | `HLS manifest acceptance full burst started`, its derived candidate count, and whether a pinned or staged cross-host candidate commits. | -| Lease reserve | The affected lease reaches its recovery/transition boundary before progress commits. | +| Lease reserve | The affected lease exhausts its known reserve while the origin remains degraded. | | Terminal preparation | `HLS terminal bundle preparation unavailable` with a typed failed or incompatible reason. | | Terminal commit | `HLS autonomous terminal owner failed closed` or `HLS terminal commit retry owner failed closed`. | | Segment/MAP/transient media | Required READY reserve cannot be established, or a required object remains unavailable. | @@ -273,7 +279,7 @@ Adapt the user and group to your deployment. | `HLS terminal bundle preparation unavailable` | Prepared media is failed or incompatible; no HTTP request attempts an on-demand TS rewrite. | | `HLS autonomous terminal owner handed off` | The endpoint handed terminal publication to the bounded autonomous commit retry owner. | | `HLS terminal commit retry owner completed` | A generation-bound retry reached a terminal completion outcome. | -| `HLS terminal commit retry owner failed closed` | Retry exhausted, missed its safe deadline, lost runtime capacity, or observed incompatible media. | +| `HLS terminal commit retry owner failed closed` | The fixed CAS/retry budget expired, runtime capacity was lost, or media was incompatible. | | `HLS segment cached` | A segment was fetched and committed to the cache. | | `HLS session switched to transient passthrough` | The manifest required transient resource handling. | diff --git a/postman/mocks/hls-window/config.yaml b/postman/mocks/hls-window/config.yaml deleted file mode 100644 index b2c649790..000000000 --- a/postman/mocks/hls-window/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: 1 -name: hls-window -interceptRequests: true -scenarios: - - name: default - path: default.js diff --git a/postman/mocks/hls-window/default.js b/postman/mocks/hls-window/default.js deleted file mode 100644 index dda1eaf7f..000000000 --- a/postman/mocks/hls-window/default.js +++ /dev/null @@ -1,35 +0,0 @@ -const http = require('http'); - -// Reuses the existing hls-window fixture. Optional controls advance origin -// metadata while a loopback streaming fixture gates actual TS responses. -http.createServer(async (req, res) => { - if (req.method === 'POST' && req.url === '/configure') { - const config = req.body; - if (!config || !/^http:\/\/127\.0\.0\.1:\d+$/.test(config.segmentOrigin) || - !Number.isSafeInteger(config.firstSequence) || config.firstSequence < 0) { - res.writeHead(400); - return res.end('invalid loopback fixture configuration'); - } - await pm.state.set('hls-window.config', config); - res.writeHead(204); - return res.end(); - } - if (req.method === 'GET' && req.url === '/index.m3u8') { - const config = await pm.state.get('hls-window.config'); - const first = config ? config.firstSequence : 0; - const segmentOrigin = config ? config.segmentOrigin + '/' : ''; - res.writeHead(200, { 'content-type': 'application/vnd.apple.mpegurl' }); - return res.end('#EXTM3U\n#EXT-X-TARGETDURATION:4\n' + - `#EXT-X-MEDIA-SEQUENCE:${first}\n` + - Array.from({ length: 6 }, (_, i) => `#EXTINF:4,\n${segmentOrigin}${first + i}.ts\n`).join('')); - } - if (req.method === 'GET' && /^\/[0-5]\.ts$/.test(req.url)) { - res.writeHead(200, { 'content-type': 'video/mp2t', 'content-length': 12 }); - return setTimeout(() => { - res.write('segment-'); - setTimeout(() => res.end('body'), 350); - }, 30); - } - res.writeHead(501); - res.end('endpoint outside the HLS window fixture contract'); -}).listen(process.env.PORT); diff --git a/postman/simulations/hls-window.sim.yaml b/postman/simulations/hls-window.sim.yaml deleted file mode 100644 index fbce063ef..000000000 --- a/postman/simulations/hls-window.sim.yaml +++ /dev/null @@ -1,5 +0,0 @@ -simulation: hls-window -routing: path -mocks: - - routeKey: origin - path: ../mocks/hls-window/config.yaml diff --git a/tools/tuliprox-testkit/src/bootstrap.rs b/tools/tuliprox-testkit/src/bootstrap.rs index 8f534f561..2873a889e 100644 --- a/tools/tuliprox-testkit/src/bootstrap.rs +++ b/tools/tuliprox-testkit/src/bootstrap.rs @@ -560,7 +560,7 @@ mod tests { #[tokio::test] async fn early_process_exit_does_not_wait_for_the_readiness_deadline() { let config = BootstrapConfig { - command: "/bin/false".to_owned(), + command: "false".to_owned(), arguments: Vec::new(), readiness_url: "http://127.0.0.1:1/ready".to_owned(), readiness_timeout_millis: 5_000,