euzu/tuliprox
1
0
Fork 0
mirror of https://github.com/euzu/tuliprox.git synced 2026-10-31 03:02:24 +01:00
Code Issues Packages Projects Releases 5 Wiki Activity

fix (hls): correct live reserve accounting and terminal cutover

Browse Source
This commit is contained in:
knylbyte
2026-10-22 10:38:06 +02:00
parent 3a451fc955
commit ea206629b6
63 changed files with 1329 additions and 1390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/src/api/endpoints/hls_api/tests/late_custom_response.rs
+86 -218
View File
@@ -1,107 +1,93 @@
use super::*;
#[tokio::test]
async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() {
async fn exhausted_fixture() -> (terminal_tail::PublicationLateFixture, String) {
let fixture = terminal_tail::publication_late_fixture().await;
let now_ms = super::super::current_time_millis();
let lease = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("published live lease");
let snapshot = lease.last_manifest_snapshot.expect("published snapshot");
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let requests_before = fixture.origin.manifest_request_count();
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert!(!response.headers().contains_key(header::LOCATION));
assert!(!response.headers().contains_key(header::RETRY_AFTER));
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1)));
assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000)));
assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n"));
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
assert!(!body.contains("/hls/shared/live/"));
let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::<Vec<_>>();
assert_eq!(media_uris.len(), 12);
let (first, second) = tokio::join!(
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
);
for response in [first, second] {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response_body(response).await, body.as_bytes());
}
let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
assert_eq!(media.status(), StatusCode::OK);
assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t");
let bytes = response_body(media).await;
assert!(!bytes.is_empty());
let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await;
assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT);
assert_eq!(response_body(range).await, bytes.slice(..188));
let router =
crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state));
let head = router
.oneshot(
Request::builder()
.method(Method::HEAD)
.uri(media_uris[0])
.extension(ConnectInfo(test_addr()))
.body(Body::empty())
.expect("HEAD request"),
)
let previous = String::from_utf8(response_body(response).await.to_vec()).unwrap();
let manager = &fixture.app_state.hls.proxy;
let now_ms = super::super::current_time_millis();
let lease = manager
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("HEAD response");
assert_eq!(head.status(), StatusCode::OK);
assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string());
assert!(response_body(head).await.is_empty());
fixture.app_state.app_config.custom_stream_response.store(None);
let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
assert_eq!(replay.status(), StatusCode::OK);
assert_eq!(response_body(replay).await, bytes);
let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease");
assert_eq!(
custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(),
StatusCode::NOT_FOUND
);
let wrong_index = media_uris[0].replace("/0.ts", "/12.ts");
assert_eq!(
custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(),
StatusCode::NOT_FOUND
);
let late = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
.unwrap();
let identity = lease.media_identity().unwrap();
let first = lease.last_manifest_snapshot.as_ref().unwrap().first_proxy_seq;
// Virtual established playback: the media clock begins with delivered bytes,
// and has consumed the complete finite READY horizon before this outage.
let started_at_ms = now_ms - 120_000;
let token = manager
.record_access_lease_segment_request_started_if_identity_matches(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
identity,
first,
started_at_ms,
)
.await
.expect("terminal lease");
assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
assert_eq!(late.playback_cursor, lease.playback_cursor);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
assert_eq!(fixture.origin.manifest_request_count(), requests_before);
.unwrap();
assert_eq!(
manager
.mark_delivered_media_for_lease(
&fixture.session,
&fixture.access_lease_id,
&fixture.proxy_session_id,
identity,
Some(token),
started_at_ms
)
.await,
tuliprox_hls::api::HlsMediaActivityCommitOutcome::Committed
);
let mut session = fixture.session.write().await;
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
session.origin_refresh.mark_started(now_ms);
drop(session);
(fixture, previous)
}
#[tokio::test]
async fn late_custom_response_without_configured_asset_retains_error_response() {
let fixture = terminal_tail::publication_late_fixture().await;
async fn expired_splice_deadline_serves_exact_last_live_uri_and_immutable_terminal_tail() {
let (fixture, previous) = exhausted_fixture().await;
let last_live = previous.lines().rfind(|line| !line.is_empty() && !line.starts_with('#')).unwrap();
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3);
let body = loop {
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
if response.status() == StatusCode::OK {
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
if body.contains("#EXT-X-ENDLIST") {
break body;
}
}
assert!(tokio::time::Instant::now() < deadline, "terminal owner must finish");
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
};
assert!(!body.contains("/cvs/hls/"));
let media = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::<Vec<_>>();
let first_csv = media.iter().position(|uri| uri.contains("/terminal/")).unwrap();
assert!(first_csv > 0);
assert_eq!(media[first_csv - 1], last_live);
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
let live = get_response(Arc::clone(&fixture.app_state), last_live, None).await;
assert_eq!(live.status(), StatusCode::OK);
assert!(!response_body(live).await.is_empty());
let csv = get_response(Arc::clone(&fixture.app_state), media[first_csv], None).await;
assert_eq!(csv.status(), StatusCode::OK);
assert!(!response_body(csv).await.is_empty());
fixture.app_state.app_config.custom_stream_response.store(None);
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let (a, b) = tokio::join!(
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None)
);
assert_eq!(response_body(a).await, body.as_bytes());
assert_eq!(response_body(b).await, body.as_bytes());
assert!(fixture.session.read().await.has_terminal_tail_protections());
}
#[tokio::test]
async fn missing_live_base_fails_explicitly_instead_of_publishing_csv_only() {
let (fixture, _) = exhausted_fixture().await;
fixture.session.write().await.segments.clear();
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(response_body(response).await.is_empty());
@@ -115,99 +101,18 @@ async fn late_custom_response_without_configured_asset_retains_error_response()
super::super::current_time_millis(),
)
.await
.expect("terminal unavailable lease");
.unwrap();
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
}
#[tokio::test]
async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() {
let (fixture, media_uri) = committed_late_http_fixture().await;
for (method, range, status) in [
(Method::GET, None, StatusCode::OK),
(Method::HEAD, None, StatusCode::OK),
(Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT),
] {
let now_ms = super::super::current_time_millis();
let previous_access_ms = now_ms.saturating_sub(50_000);
assert!(fixture
.app_state
.hls
.proxy
.activate_access_lease(
&fixture.access_lease_id,
&fixture.proxy_session_id,
previous_access_ms,
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let before = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("valid late lease");
let response =
custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await;
assert_eq!(response.status(), status);
let after = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.expect("renewed late lease");
assert!(after.last_seen_at_ms >= now_ms);
assert!(after.valid_until_ms > before.valid_until_ms);
assert!(after.active_until_ms > before.active_until_ms);
assert_eq!(after.playback_mode, before.playback_mode);
assert_eq!(after.playback_cursor, before.playback_cursor);
assert_eq!(after.admission_generation, before.admission_generation);
}
}
#[tokio::test]
async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() {
let (fixture, media_uri) = committed_late_http_fixture().await;
let now_ms = super::super::current_time_millis();
assert!(fixture
.app_state
.hls
.proxy
.activate_access_lease(
&fixture.access_lease_id,
&fixture.proxy_session_id,
now_ms - 50_000,
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let before = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("valid late lease");
let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint");
for (uri, range, expected) in [
(media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND),
(media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND),
(media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND),
(media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE),
] {
let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await;
assert_eq!(response.status(), expected);
}
fixture.app_state.app_config.api_proxy.store(None);
let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await;
assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status());
let after = fixture
async fn missing_csv_asset_keeps_the_explicit_error_response() {
let (fixture, _) = exhausted_fixture().await;
fixture.app_state.app_config.custom_stream_response.store(None);
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(response_body(response).await.is_empty());
let lease = fixture
.app_state
.hls
.proxy
@@ -217,43 +122,6 @@ async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease()
super::super::current_time_millis(),
)
.await
.expect("unchanged late lease");
assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms);
assert_eq!(after.valid_until_ms, before.valid_until_ms);
assert_eq!(after.active_until_ms, before.active_until_ms);
}
async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) {
let fixture = terminal_tail::publication_late_fixture().await;
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned();
(fixture, media_uri)
}
async fn custom_media_response(app_state: Arc<AppState>, uri: &str, range: Option<&str>) -> Response<Body> {
custom_media_response_with_method(app_state, uri, range, Method::GET).await
}
async fn custom_media_response_with_method(
app_state: Arc<AppState>,
uri: &str,
range: Option<&str>,
method: Method,
) -> Response<Body> {
let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state);
let mut request = Request::builder().method(method).uri(uri);
if let Some(range) = range {
request = request.header(header::RANGE, range);
}
router
.oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request"))
.await
.expect("custom media response")
.unwrap();
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
}
backend/app/src/api/endpoints/hls_api/tests/manifest_short_window.rs
+12 -7
View File
@@ -141,20 +141,25 @@ async fn wait_for_shared_first_ready_refresh(
fixture: &ShortWindowFixture,
session: &HlsSessionHandle,
previous_requests: usize,
expected_generation: u64,
) {
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if fixture.origin.manifest_request_count() > previous_requests
&& !session.read().await.origin_refresh.in_flight
{
let refresh_in_flight = {
let session = session.read().await;
// Later leases may start another ordinary freshness fetch after a fast
// response; they must still share the completed qualification.
assert_eq!(session.origin_control.acceptance_generation.0, expected_generation);
session.origin_refresh.in_flight
};
if fixture.origin.manifest_request_count() > previous_requests && !refresh_in_flight {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("additional leases complete one shared freshness request");
assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1);
.expect("additional leases complete freshness checks without requalification");
}
async fn assert_first_ready_lease_media(
@@ -217,7 +222,7 @@ async fn first_ready_leases_share_qualification_while_first_segment_is_loading()
let fixture = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await });
}
wait_for_shared_first_ready_refresh(&fixture, &session, 3).await;
wait_for_shared_first_ready_refresh(&fixture, &session, 3, generation.0).await;
assert!(session.read().await.last_rendered_manifest.is_none());
assert_eq!(session.read().await.origin_control.acceptance_generation, generation);
gate.close();
@@ -316,7 +321,7 @@ async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loa
let fixture = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await });
}
wait_for_shared_first_ready_refresh(&fixture, &session, 5).await;
wait_for_shared_first_ready_refresh(&fixture, &session, 5, 2).await;
assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2);
assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit));
gate.close();
backend/app/src/api/endpoints/hls_api/tests/reduced_retry.rs
+10
View File
@@ -153,6 +153,16 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) {
"{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}",
binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0
);
{
let mut leases = app.hls.proxy.access_leases().write().await;
let mut active = leases.remove_access_lease(&HlsAccessLeaseId(lease.to_string())).unwrap();
let start_seq = active.playback_cursor.first_segment_started_proxy_seq.unwrap();
let state = session.read().await;
let start = state.segments[&start_seq].media_start_ms;
let horizon = state.segments.values().map(|entry| entry.media_start_ms + entry.duration_ms).max().unwrap();
active.playback_cursor.first_segment_started_at_ms = Some(current_time_millis() - (horizon - start - 8_000));
leases.prepare_access_lease(active);
}
session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure;
let directive =
match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id)
backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs
+15 -6
View File
@@ -694,6 +694,10 @@ pub(in crate::api::endpoints::hls_api::tests) async fn prepare_terminal_cutover_
(asset_buffer, asset)
}
#[allow(
clippy::too_many_lines,
reason = "One encrypted playback fixture includes real delivery, elapsed media time and prepared terminal bytes."
)]
pub(in crate::api::endpoints::hls_api::tests) async fn prepared_terminal_cutover_fixture(
) -> PreparedTerminalCutoverFixture {
let temp_dir = tempfile::tempdir().expect("terminal cutover tempdir");
@@ -778,6 +782,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn prepared_terminal_cutover
assert_eq!(response.status(), StatusCode::OK);
assert!(!response_body(response).await.is_empty());
}
{
let mut leases = app_state.hls.proxy.access_leases().write().await;
let mut lease = leases.remove_access_lease(&lease_id).unwrap();
lease.playback_cursor.first_segment_started_at_ms =
lease.playback_cursor.first_segment_started_at_ms.map(|at| at - 10_000);
leases.prepare_access_lease(lease);
}
PreparedTerminalCutoverFixture {
_temp_dir: temp_dir,
origin,
@@ -923,12 +934,10 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_
) -> (u64, u64) {
let cutover_now_ms = pressured_lease
.playback_cursor
.first_segment_completed_at_ms
.expect("measured lease playback start")
// Enter the acquisition window while the live base and its key are still usable.
.saturating_add(
fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms),
);
.first_segment_started_at_ms
.expect("actual transfer start")
.saturating_add(fixture.base_manifest.playlist_duration_ms)
.saturating_add(3_800);
let first = commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.app_state.hls_ctx(),
backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs
+28 -26
View File
@@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path()
}
#[tokio::test]
async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() {
async fn commits_prepared_terminal_tail_once_after_complete_reserve_consumption() {
let fixture = prepared_terminal_cutover_fixture().await;
let directive = terminal_cutover_acceptance_directive(&fixture).await;
let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await;
@@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhaus
}
#[tokio::test]
async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() {
async fn warm_fmp4_at_exhaustion_preserves_the_explicit_incompatible_media_error() {
let temp_dir = tempfile::tempdir().expect("tempdir");
let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)));
enable_channel_unavailable_custom_response(&app_state);
@@ -143,39 +143,41 @@ async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splic
};
publish_test_manifest_and_exhaust_configured_acceptance(&app_state, &proxy_session_id, &lease_id, snapshot, now_ms)
.await;
let manager = &app_state.hls.proxy;
let session = manager.sessions().get_by_proxy_session_id(&proxy_session_id).await.unwrap();
let identity = manager
.access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms)
.await
.unwrap()
.media_identity()
.unwrap();
let started_at = now_ms - 120_000;
let token = manager
.record_access_lease_segment_request_started_if_identity_matches(
&lease_id,
&proxy_session_id,
identity,
base_proxy_seq,
started_at,
)
.await
.unwrap();
let _ = manager
.mark_delivered_media_for_lease(&session, &lease_id, &proxy_session_id, identity, Some(token), started_at)
.await;
let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0);
let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert!(!response.headers().contains_key(header::LOCATION));
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest");
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
assert!(body.contains("/cvs/hls/"));
assert!(!body.contains("#EXT-X-MAP"));
assert!(!body.contains("/hls/shared/live/"));
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(response_body(response).await.is_empty());
let lease = app_state
.hls
.proxy
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
.await
.expect("late custom lease snapshot");
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
assert_eq!(repeated.status(), StatusCode::OK);
assert_eq!(response_body(repeated).await, body.as_bytes());
let repeated_lease = app_state
.hls
.proxy
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
.await
.expect("stable terminal custom lease");
assert_eq!(repeated_lease.playback_mode, lease.playback_mode);
let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0);
let terminal_response = get_response(app_state, &terminal_uri, None).await;
assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND);
assert!(!terminal_response.headers().contains_key(header::LOCATION));
assert!(response_body(terminal_response).await.is_empty());
.unwrap();
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
}
#[tokio::test]
backend/app/tests/hls_manifest_window.rs
-199
View File
@@ -1,199 +0,0 @@
//! Real process regression against the existing local Postman hls-window fixture.
//!
//! Start its simulation with a fixed port and an NDJSON log:
//! `/Users/philippmielke/.local/bin/postman simulation run postman/simulations/hls-window.sim.yaml --port 4926 --no-history --output ndjson`
//!
//! Run through `./bin/test.sh -p tuliprox --test hls_manifest_window -- --ignored`,
//! setting `TULIPROX_HLS_WINDOW_SIMULATION_ORIGIN=http://127.0.0.1:4926/origin`
//! and `TULIPROX_HLS_WINDOW_EVIDENCE_DIR` to an absolute output directory.
//! The member serves real HLS metadata; a loopback TS origin gates new bytes.
//! Only the URL-free client protocol is suitable for sharing; keep raw logs local.
use std::{
io::Write as _,
process::Stdio,
sync::{
atomic::{AtomicUsize, Ordering},
Arc,
},
time::Duration,
};
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
net::TcpListener,
sync::watch,
};
const BODY: &[u8] =
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"));
fn media_url(base: &str, playlist: &str) -> String {
let uri = playlist.lines().find(|line| !line.is_empty() && !line.starts_with('#')).unwrap();
reqwest::Url::parse(base).unwrap().join(uri).unwrap().to_string()
}
fn sequence(body: &str, tag: &str) -> u64 {
body.lines().find_map(|line| line.strip_prefix(tag)).unwrap_or("0").parse().unwrap()
}
#[tokio::test]
#[allow(
clippy::too_many_lines,
reason = "Keep the real process, streaming gate and HTTP assertions in one isolated scenario."
)]
#[ignore = "requires local Postman hls-window simulation; see module documentation"]
async fn live_window_http_republishes_same_lease_while_successor_is_gated() {
let simulation = std::env::var("TULIPROX_HLS_WINDOW_SIMULATION_ORIGIN").expect("Postman origin URL");
let evidence =
std::path::PathBuf::from(std::env::var("TULIPROX_HLS_WINDOW_EVIDENCE_DIR").expect("evidence directory"));
std::fs::create_dir_all(&evidence).unwrap();
let mut protocol = std::fs::File::create(evidence.join("client.ndjson")).unwrap();
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
for mode in ["first_ready", "progressive", "optimistic"] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let segment_origin = format!("http://{}", listener.local_addr().unwrap());
let configure = |first| {
client
.post(format!("{simulation}/configure"))
.json(&serde_json::json!({"firstSequence": first, "segmentOrigin": segment_origin}))
};
assert_eq!(configure(0).send().await.unwrap().status(), 204);
let catalog = format!(
"#EXTM3U\n#EXTINF:-1 tvg-id=\"window\" group-title=\"Test\",Window Channel\n{simulation}/index.m3u8\n"
);
let (release, released) = watch::channel(false);
let attempts = Arc::new(AtomicUsize::new(0));
let counted = attempts.clone();
let origin = tokio::spawn(async move {
let mut children = tokio::task::JoinSet::new();
loop {
tokio::select! {
accepted = listener.accept() => {
let (socket, _) = accepted.unwrap();
let catalog = catalog.clone();
let mut released = released.clone();
let attempts = counted.clone();
children.spawn(async move {
let mut reader = BufReader::new(socket);
let mut request = String::new(); reader.read_line(&mut request).await.unwrap();
let path = request.split_whitespace().nth(1).unwrap().to_string();
loop {
let mut line = String::new(); reader.read_line(&mut line).await.unwrap();
if line == "\r\n" || line.is_empty() { break; }
}
let mut socket = reader.into_inner();
let body = if path == "/catalog.m3u" { catalog.as_bytes() } else { BODY };
let seq = path.strip_prefix('/').and_then(|path| path.strip_suffix(".ts"))
.and_then(|seq| seq.parse::<u64>().ok());
if seq.is_some_and(|seq| seq >= 6) {
attempts.fetch_add(1, Ordering::AcqRel);
while !*released.borrow_and_update() {
if released.changed().await.is_err() { return; }
}
}
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
if socket.write_all(headers.as_bytes()).await.is_ok() { let _ = socket.write_all(body).await; }
});
}
_ = children.join_next(), if !children.is_empty() => {}
}
}
});
let directory = tempfile::tempdir().unwrap();
let root = directory.path();
for name in ["web", "storage", "backup"] {
std::fs::create_dir(root.join(name)).unwrap();
}
let bound = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = bound.local_addr().unwrap().port();
drop(bound);
let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n hls_cache:\n cache_path: '{}'\n strip:\n mode: segments\n value: 0\n origin_segment_timeout_ms: 10000\n max_concurrent_segment_fetches_per_session: 6\n initial_manifest_wait_timeout_secs: 3\n startup:\n mode: {mode}\n segment_repair:\n max_level: off\n", root.join("web").display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display());
let sources = format!("inputs:\n - name: window-origin\n type: m3u\n url: {segment_origin}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [window-origin]\n targets:\n - name: window\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n");
let users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Window test\nuser:\n - target: window\n credentials:\n - username: viewer\n password: test-password\n proxy: reverse\n server: local\n status: Active\n max_connections: 1\n");
for (name, body) in [("config.yml", config), ("source.yml", sources), ("api-proxy.yml", users)] {
std::fs::write(root.join(name), body).unwrap();
}
let log_file = evidence.join(format!("sut-{mode}.log"));
let log = std::fs::File::create(&log_file).unwrap();
let mut process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox"))
.args(["-s", "-c", "config.yml", "-i", "source.yml", "-a", "api-proxy.yml"])
.current_dir(root)
.env("RUST_LOG", "debug")
.stdout(Stdio::from(log.try_clone().unwrap()))
.stderr(Stdio::from(log))
.kill_on_drop(true)
.spawn()
.unwrap();
let base = format!("http://127.0.0.1:{port}");
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
let entry = loop {
assert!(
process.try_wait().unwrap().is_none(),
"SUT exited: {}",
std::fs::read_to_string(&log_file).unwrap()
);
if let Ok(response) = client.get(format!("{base}/m3u?username=viewer&password=test-password")).send().await
{
if let Ok(body) = response.text().await {
if let Some(url) = body.lines().find(|line| line.starts_with("http")) {
break url.to_owned();
}
}
}
assert!(tokio::time::Instant::now() < deadline, "catalog startup timeout");
tokio::time::sleep(Duration::from_millis(25)).await;
};
let master = client.get(&entry).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
let variant = media_url(&entry, &master);
let initial = client.get(&variant).send().await.unwrap();
assert_eq!(initial.status(), 200);
let previous = initial.text().await.unwrap();
writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"initial","status":200,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap();
let segment = client.get(media_url(&variant, &previous)).send().await.unwrap();
assert_eq!(segment.status(), 200);
assert!(!segment.bytes().await.unwrap().is_empty());
assert_eq!(configure(6).send().await.unwrap().status(), 204);
let deadline = tokio::time::Instant::now() + Duration::from_secs(8);
while attempts.load(Ordering::Acquire) == 0 {
let response = client.get(&variant).send().await.unwrap();
assert_eq!(response.status(), 200, "{mode}: waiting for origin refresh");
assert_eq!(response.text().await.unwrap(), previous);
writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"refresh","status":200,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap();
assert!(tokio::time::Instant::now() < deadline, "successor origin request was never made");
tokio::time::sleep(Duration::from_millis(25)).await;
}
for poll in 0..20 {
let response = client.get(&variant).send().await.unwrap();
let status = response.status().as_u16();
assert_eq!(status, 200, "{mode} held poll {poll}");
assert!(response.headers().get("retry-after").is_none());
assert_eq!(response.text().await.unwrap(), previous);
writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"held","poll":poll,"status":status,"mediaSequence":sequence(&previous,"#EXT-X-MEDIA-SEQUENCE:"),"successorOriginRequests":attempts.load(Ordering::Acquire)})).unwrap();
}
let retained_responses = std::fs::read_to_string(&log_file)
.unwrap()
.lines()
.filter(|line| line.contains("HLS live window retained:"))
.count();
assert!(retained_responses >= 20, "{mode}: held HTTP requests must exercise the retained-window branch");
release.send(true).unwrap();
let deadline = tokio::time::Instant::now() + Duration::from_secs(8);
loop {
let response = client.get(&variant).send().await.unwrap();
assert_eq!(response.status(), 200);
let next = response.text().await.unwrap();
if sequence(&next, "#EXT-X-MEDIA-SEQUENCE:") > sequence(&previous, "#EXT-X-MEDIA-SEQUENCE:") {
assert!(
sequence(&next, "#EXT-X-DISCONTINUITY-SEQUENCE:")
>= sequence(&previous, "#EXT-X-DISCONTINUITY-SEQUENCE:")
);
writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"advanced","status":200,"mediaSequence":sequence(&next,"#EXT-X-MEDIA-SEQUENCE:"),"successorOriginRequests":attempts.load(Ordering::Acquire),"retainedResponses":retained_responses})).unwrap();
break;
}
writeln!(protocol, "{}", serde_json::json!({"mode":mode,"phase":"released","status":200,"mediaSequence":sequence(&next,"#EXT-X-MEDIA-SEQUENCE:")})).unwrap();
assert!(tokio::time::Instant::now() < deadline, "successor did not publish");
tokio::time::sleep(Duration::from_millis(25)).await;
}
process.kill().await.unwrap();
origin.abort();
}
}
backend/dvr/src/recording/recording_worker.rs
+2 -6
View File
@@ -991,12 +991,8 @@ mod tests {
Ok::<_, std::io::Error>(())
});
let wrapper = directory.path().join("ffmpeg-with-proxy");
// Explicitly remove proxy exclusions so localhost still exercises the bypass.
tokio::fs::write(
&wrapper,
"#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n",
)
.await?;
// Keep the listener exclusions supplied by the worker while adding an unusable proxy.
tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?;
fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?;
let mut recording = make_recording(chrono::Utc::now().timestamp(), 60);
recording.url = url;
backend/hls/src/availability/mod.rs
+7 -3
View File
@@ -42,7 +42,6 @@ pub use self::{
};
use std::future::Future;
pub const HLS_PLAYBACK_RATE_GUARD_MILLI: u16 = 1_050;
const HLS_AVAILABILITY_STATE_ACCESS_ATTEMPTS: usize = 3;
const HLS_AVAILABILITY_REEVALUATION_MAX_ATTEMPTS: u8 = 64;
const HLS_AVAILABILITY_REEVALUATION_DEADLINE_MS: u64 = 2_000;
@@ -169,13 +168,18 @@ pub async fn hls_startup_admission_allows_snapshot(
snapshot: &HlsLeaseManifestSnapshot,
now_ms: u64,
) -> bool {
let (ready_timeline, origin_state, observed_latency) = {
let (ready_timeline, origin_state, observed_latency, target_duration_ms) = {
let session = session.read().await;
let degraded = session.origin_control.path_condition.is_degraded();
(
session.ready_timeline_snapshot(snapshot.first_proxy_seq, now_ms),
if degraded { HlsStartupAdmissionOriginState::Degraded } else { HlsStartupAdmissionOriginState::Healthy },
session.origin_control.recovery_samples.latency_snapshot(),
session
.origin_control
.target_duration_snapshot_ms
.or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000)))
.unwrap_or(snapshot.target_duration_ms),
)
};
let workload = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling();
@@ -190,7 +194,7 @@ pub async fn hls_startup_admission_allows_snapshot(
origin_state,
recovery_trigger_budget: recovery_trigger_budget(
pressure_policy,
snapshot.target_duration_ms,
target_duration_ms,
workload,
observed_latency,
),
backend/hls/src/availability/recovery_pressure.rs
+3 -4
View File
@@ -5,7 +5,7 @@
//! timing seed the acceptance episode starts from. Committed back onto the
//! session under a CAS loop, so a concurrent refresh cannot lose the decision.
use super::{hls_recovery_timing_policy, HLS_PLAYBACK_RATE_GUARD_MILLI};
use super::hls_recovery_timing_policy;
use crate::{
hls_ctx::HlsCtx,
lease::HlsAccessLeaseStore,
@@ -207,13 +207,12 @@ pub(super) fn evaluate_and_commit_session_recovery_pressure(
now_ms,
);
let recovery_trigger_budget =
recovery_trigger_budget(policy, manifest.target_duration_ms, workload, observed_latency);
recovery_trigger_budget(policy, publication_target_duration_ms, workload, observed_latency);
let reserve = evaluate_lease_reserve(HlsLeaseReserveInput {
manifest,
cursor: &lease.playback_cursor,
ready_timeline: &ready_timeline,
now_ms,
playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget,
origin_path_degraded: effective_origin_degraded,
recovery_committed: false,
@@ -227,7 +226,7 @@ pub(super) fn evaluate_and_commit_session_recovery_pressure(
reserve,
cursor: lease.playback_cursor.clone(),
workload,
target_duration_ms: manifest.target_duration_ms,
target_duration_ms: publication_target_duration_ms,
latest_safe_terminal_commit_at: cutover_timing.latest_safe_terminal_commit_at,
recovery_boundary_slack_ms: HlsRecoveryBoundarySlackMs::from_reserve_and_boundary(
reserve.guaranteed_reserve_ms,
backend/hls/src/availability/terminal_cutover.rs
+19 -56
View File
@@ -8,12 +8,10 @@
use super::{
recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget},
terminal_pending_owner::{
commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure,
commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner,
terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback,
},
HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution,
HLS_PLAYBACK_RATE_GUARD_MILLI,
};
use crate::{
hls_ctx::HlsCtx,
@@ -85,7 +83,6 @@ struct HlsLeaseTerminalDecisionContext<'a> {
state: &'a HlsLeaseCutoverStateSnapshot,
evaluation: HlsLeaseTerminalCutoverEvaluation,
now_ms: u64,
purpose: HlsTerminalDecisionPurpose,
}
impl<'a> HlsLeaseTerminalDecisionContext<'a> {
@@ -124,13 +121,12 @@ fn evaluate_lease_terminal_cutover(
state.session_target_duration,
);
let recovery_trigger_budget =
recovery_trigger_budget(pressure_policy, manifest.target_duration_ms, workload, state.observed_latency);
recovery_trigger_budget(pressure_policy, state.target_duration_ms, workload, state.observed_latency);
let reserve = evaluate_lease_reserve(HlsLeaseReserveInput {
manifest,
cursor: &lease.playback_cursor,
ready_timeline: &state.ready_timeline,
now_ms,
playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget,
origin_path_degraded,
recovery_committed: false,
@@ -163,39 +159,14 @@ fn evaluate_lease_terminal_cutover(
}
}
async fn resolve_terminal_cutover_before_commit_window(
fn resolve_terminal_cutover_before_commit_window(
context: &HlsLeaseTerminalDecisionContext<'_>,
) -> HlsDetailedTerminalResolution {
if !context.evaluation.origin_path_degraded {
return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed);
}
if matches!(context.purpose, HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback) {
let Some(preparation) = context
.ctx
.hls_proxy
.prepare_access_lease_terminal_unavailable_after_owner_failure(context.preparation_request())
.await
else {
return HlsDetailedTerminalResolution::with_deadline(
HlsTerminalResolution::Reevaluate,
context.evaluation.safe_deadline,
);
};
return HlsDetailedTerminalResolution::with_deadline(
commit_prepared_terminal_unavailable_after_owner_failure(
context.ctx,
context.session,
context.proxy_session_id,
&context.lease.lease_id,
&preparation,
context.now_ms,
HlsTerminalTailCompatibility::TerminalMediaNotReady,
),
context.evaluation.safe_deadline,
);
}
let Some(live_reserve_deadline) = context.evaluation.safe_deadline else {
return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::Reevaluate);
return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed);
};
HlsDetailedTerminalResolution {
resolution: HlsTerminalResolution::LiveAllowed,
@@ -306,6 +277,12 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai
return HlsDetailedTerminalResolution::resolved(HlsTerminalResolution::LiveAllowed);
}
let evaluation = evaluate_lease_terminal_cutover(ctx, lease, manifest, &state, now_ms);
log::debug!("HLS terminal reserve decision: proxy_session={} lease={} start_at_ms={:?} position_known={} position_ms={} reserve_ms={} session_target_duration_ms={} snapshot_generation={} may_publish={}",
crate::safe_proxy_session_id(proxy_session_id), crate::safe_hls_access_lease_id(&lease.lease_id),
lease.playback_cursor.first_segment_started_at_ms, evaluation.reserve.playback_position_known,
evaluation.reserve.conservative_playback_position_ms, evaluation.reserve.guaranteed_reserve_ms,
state.target_duration_ms, manifest.snapshot_generation,
evaluation.origin_path_degraded && evaluation.reserve.is_exhausted());
{
let mut session = session.write().await;
if session.origin_control.progress_generation == state.progress_generation
@@ -320,7 +297,7 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai
);
}
}
let commit_window = evaluation.commit_window;
let may_publish = evaluation.origin_path_degraded && evaluation.reserve.is_exhausted();
let decision_context = HlsLeaseTerminalDecisionContext {
ctx,
session,
@@ -330,12 +307,11 @@ pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detai
state: &state,
evaluation,
now_ms,
purpose,
};
if commit_window == HlsTerminalCommitWindow::NotDue {
resolve_terminal_cutover_before_commit_window(&decision_context).await
} else {
if may_publish {
commit_terminal_cutover(&decision_context).await
} else {
resolve_terminal_cutover_before_commit_window(&decision_context)
}
}
@@ -356,7 +332,11 @@ async fn snapshot_lease_cutover_state(
progress_generation: session.origin_control.progress_generation,
media_readiness_generation: session.activity.media_readiness_generation,
last_media_progress_at_ms: session.origin_control.last_media_progress_at_ms,
target_duration_ms: session.origin_control.target_duration_snapshot_ms.unwrap_or(manifest_target_duration_ms),
target_duration_ms: session
.origin_control
.target_duration_snapshot_ms
.or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000)))
.unwrap_or(manifest_target_duration_ms),
session_target_duration: session.target_duration,
observed_latency: session.origin_control.recovery_samples.latency_snapshot(),
}
@@ -370,19 +350,6 @@ async fn commit_prepared_terminal_decision(
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
) -> HlsTerminalResolution {
if !preparation.trigger.is_runtime_policy()
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
{
return commit_late_custom_response(HlsTerminalCommitContext {
ctx,
session,
proxy_session_id,
lease_id,
preparation,
now_ms,
})
.await;
}
let reason = preparation.trigger.reason();
let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) {
Ok(asset) => asset,
@@ -408,11 +375,7 @@ pub async fn commit_prepared_runtime_custom_tail(
if preparation.trigger.reason() != asset.reason {
return HlsTerminalResolution::Reevaluate;
}
if !preparation.trigger.is_runtime_policy()
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
{
return commit_late_custom_response(context).await;
}
let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset);
let media_asset = asset.asset;
let static_incompatibility = if preparation.manifest_snapshot.delivery_mode
backend/hls/src/availability/terminal_pending_owner.rs
+6 -48
View File
@@ -66,8 +66,7 @@ pub(super) async fn register_terminal_pending_owner(
let Some(owner_key) = terminal_pending_owner_key(context, expected_asset, bundle_key) else {
return HlsTerminalResolution::Reevaluate;
};
let latest_safe_commit_at_ms =
context.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let latest_safe_commit_at_ms = owner_key.latest_safe_commit_at_ms;
let coordinator = context.ctx.hls_proxy.terminal_pending();
let task_ctx = context.ctx.clone();
let task_session = Arc::clone(context.session);
@@ -109,11 +108,11 @@ fn terminal_pending_owner_key(
decision_generation: context.preparation.decision_generation,
reason: expected_asset.reason,
bundle_key,
latest_safe_commit_at_ms: context
.preparation
.cutover_timing
.latest_safe_terminal_commit_at
.as_millis_since_epoch(),
latest_safe_commit_at_ms: if context.preparation.trigger.is_runtime_policy() {
context.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
} else {
context.now_ms.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis())
},
})
}
@@ -388,9 +387,6 @@ pub(super) async fn terminal_resolution_with_failed_closed_fallback(
if context.preparation.trigger.is_runtime_policy() {
return resolution;
}
if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed {
return commit_late_custom_response(context).await;
}
commit_prepared_terminal_unavailable_after_owner_failure(
context.ctx,
context.session,
@@ -402,44 +398,6 @@ pub(super) async fn terminal_resolution_with_failed_closed_fallback(
)
}
pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution {
let prepared = crate::late_custom_response::prepare_late_custom_response(
context.ctx,
context.proxy_session_id,
context.lease_id,
context.preparation,
context.now_ms,
)
.await;
let (plan, expected_asset) = match prepared {
Ok(prepared) => prepared,
Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => {
return HlsTerminalResolution::Reevaluate
}
Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => {
return commit_prepared_terminal_unavailable_after_owner_failure(
context.ctx,
context.session,
context.proxy_session_id,
context.lease_id,
context.preparation,
context.now_ms,
reason,
)
}
};
let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
session: context.session,
lease_id: context.lease_id,
proxy_session_id: context.proxy_session_id,
preparation: context.preparation,
now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms),
payload: HlsTerminalCommitPayload::CustomResponse(plan),
asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)),
});
terminal_resolution_for_commit_outcome(outcome, context.now_ms)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum HlsAutonomousTerminalObservation {
Committed,
backend/hls/src/availability/tests.rs
+2 -1
View File
@@ -225,7 +225,8 @@ mod terminal_fixtures;
use terminal_fixtures::{
assert_post_refresh_registration_failure_leaves_no_unowned_live_lease,
assert_terminal_pending_registration_failure_commits_unavailable, terminal_base_without_timestamps,
terminal_pending_commit_fixture, terminal_pending_commit_fixture_with_base, terminal_pending_commit_reserve,
terminal_pending_commit_fixture, terminal_pending_commit_fixture_with_base,
terminal_pending_commit_fixture_with_cache, terminal_pending_commit_reserve,
};
mod pressure_fixtures;
use pressure_fixtures::{
backend/hls/src/availability/tests/late_custom_response.rs
+127 -468
View File
@@ -21,7 +21,12 @@ async fn prepare(
fixture.now_ms,
)
.await
.unwrap_or_else(|_| panic!("late custom plan"))
.unwrap_or_else(|error| match error {
crate::late_custom_response::HlsLateCustomResponseError::Superseded => panic!("late plan superseded"),
crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason) => {
panic!("late plan unavailable: {reason:?}")
}
})
}
fn commit(
@@ -42,491 +47,145 @@ fn commit(
}
#[tokio::test]
async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() {
let fixture = late_fixture("late-without-base").await;
let (plan, asset) = prepare(&fixture).await;
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media);
assert!(
fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms)
.await
.is_err(),
"preparation cannot authorize uncommitted custom media"
);
fixture.session.write().await.segments.clear();
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n"));
assert!(!plan.manifest_body().contains("/hls/shared/live/"));
let segment = fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms)
.await
.expect("committed late media");
assert!(!segment.bytes.is_empty());
fixture.ctx.app_config.custom_stream_response.store(None);
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted);
let replay = fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1)
.await
.expect("frozen media after config reload");
assert_eq!(replay.bytes, segment.bytes);
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms)
.await
.is_err());
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms)
.await
.is_err());
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000)
.await
.is_err());
}
#[tokio::test]
async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() {
for changed_asset in [false, true] {
let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await;
let (plan, asset) = prepare(&fixture).await;
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let expected = if changed_asset {
fixture.ctx.app_config.custom_stream_response.store(None);
HlsTerminalCommitOutcome::SupersededGeneration
} else {
fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000);
HlsTerminalCommitOutcome::RecoveryCommitted
};
assert_eq!(commit(&fixture, plan, asset, now_ms), expected);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("lease");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
}
}
#[tokio::test]
async fn late_custom_response_preserves_the_prepared_asset_reason_guard() {
let fixture = late_fixture("late-wrong-asset-reason").await;
let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted)
.expect("configured alternate custom asset");
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(
commit_prepared_runtime_custom_tail(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
&fixture.preparation,
now_ms,
asset,
)
.await,
HlsTerminalResolution::Reevaluate
);
async fn legacy_csv_only_plan_is_rejected_even_after_splice_deadline() {
let fixture = late_fixture("legacy-late-plan-rejected").await;
let (plan, expected_asset) = prepare(&fixture).await;
assert_eq!(commit(&fixture, plan, expected_asset, fixture.now_ms), HlsTerminalCommitOutcome::BundleIncompatible);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
}
#[tokio::test]
async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() {
let mut fixture = late_fixture("late-target-duration-error").await;
let mut manifest = fixture.preparation.manifest_snapshot.clone();
manifest.target_duration_ms = 1_000;
manifest.playlist_duration_ms = 1_000;
manifest.last_visible_media_end_ms = 1_000;
Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000;
let guard = fixture
.ctx
.hls_proxy
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("publication guard");
assert!(fixture
.ctx
.hls_proxy
.commit_access_lease_manifest_publication(
&fixture.lease_id,
&fixture.proxy_session_id,
guard,
manifest,
fixture.now_ms,
)
.await
.is_committed());
async fn exhausted_lease_uses_last_live_segment_after_old_splice_deadline() {
let fixture = late_fixture("late-tail-keeps-live-base").await;
assert!(
fixture.now_ms >= fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
);
let previous = fixture.preparation.manifest_snapshot.visible_segments.last().unwrap().clone();
let context = HlsTerminalCommitContext {
ctx: &fixture.ctx,
session: &fixture.session,
proxy_session_id: &fixture.proxy_session_id,
lease_id: &fixture.lease_id,
preparation: &fixture.preparation,
now_ms: fixture.now_ms,
};
let bundle = fixture.ready_bundle();
let outcome = super::super::terminal_cutover::commit_ready_terminal_bundle(
context,
Arc::clone(&fixture.asset),
fixture.expected_asset,
fixture.bundle_key,
bundle,
)
.await;
assert_eq!(outcome, HlsTerminalCommitOutcome::Committed);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("lease with short target duration");
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot");
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
let expected = HlsTerminalTailCompatibility::TargetDurationExceeded {
asset_ms: fixture.asset.duration_ms(),
target_ms: 1_000,
.unwrap();
let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else {
panic!("live tail required");
};
assert!(matches!(prepare_late_custom_response(
&fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms,
).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected));
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(
super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext {
assert_eq!(plan.base_manifest.visible_segments.last(), Some(&previous));
let manifest = terminal_tail_manifest_body(&plan, &fixture.proxy_session_id, &fixture.lease_id).unwrap();
let live = manifest.find(previous.uri.as_ref()).expect("last live URI");
let terminal = manifest.find("/terminal/").expect("CSV tail");
assert!(live < terminal);
assert!(manifest.ends_with("#EXT-X-ENDLIST\n"));
assert!(!manifest.contains("/cvs/hls/"));
fixture.ctx.hls_proxy.run_garbage_collection_once(fixture.now_ms + 1).await.unwrap();
let key = SegmentCacheKey::new(fixture.proxy_session_id.clone(), previous.proxy_seq, "ts");
assert!(fixture.ctx.hls_proxy.segment_cache().metadata(&key).await.unwrap().is_some());
}
#[tokio::test]
#[allow(
clippy::too_many_lines,
reason = "Exercise real expiry GC before and after the atomic live-to-terminal protection transfer."
)]
async fn gc_expires_control_bytes_but_keeps_the_last_live_anchor_through_terminal_commit() {
let directory = tempfile::tempdir().unwrap();
let mut fixture = terminal_pending_commit_fixture_with_cache("gc-live-terminal-anchor", directory.path()).await;
let manager = &fixture.ctx.hls_proxy;
let control = SegmentCacheKey::new(fixture.proxy_session_id.clone(), 39, "ts");
manager.segment_cache().write_bytes_and_commit(&control, TERMINAL_ASSET_BYTES).await.unwrap();
{
let mut state = fixture.session.write().await;
let mut entry = state.segments[&40].clone();
entry.proxy_seq = 39;
entry.cache_key = control.clone();
entry.origin_key.host_local_sequence = 39;
state.segments.insert(39, entry);
state.proxy_next_seq = Some(41);
let OriginManifestParseOutcome::Normal(origin) = parse_origin_media_manifest(
"#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:12,\n100.ts\n#EXTINF:12,\n101.ts\n#EXTINF:12,\n102.ts\n",
"http://origin.example/index.m3u8",
) else { panic!("normal origin"); };
state.apply_origin_manifest_for_host(&origin, 1).unwrap();
state.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure;
}
let gc = crate::HlsGarbageCollector::new(
Arc::clone(manager.sessions()),
Arc::clone(manager.segment_cache()),
crate::GarbageCollectionPolicy {
cache_duration_ms: 1,
session_idle_timeout_ms: 1_000_000,
..crate::GarbageCollectionPolicy::default()
},
String::new(),
);
gc.install_access_leases(manager.access_leases());
let report = gc.run_once(fixture.now_ms + 30_000).await.unwrap();
assert!(
manager.segment_cache().metadata(&control).await.unwrap().is_none(),
"control must actually be reclaimed: {report:?}"
);
assert!(manager.segment_cache().metadata(&control).await.unwrap().is_none());
let live_key = SegmentCacheKey::new(fixture.proxy_session_id.clone(), 40, "ts");
assert!(manager.segment_cache().metadata(&live_key).await.unwrap().is_some());
let state = fixture.session.read().await;
let reserve = fixture.preparation.reserve;
let preparation = manager.prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest {
lease_id: &fixture.lease_id,
proxy_session_id: &fixture.proxy_session_id,
manifest_snapshot_generation: fixture.preparation.manifest_snapshot_generation,
cursor_generation: fixture.preparation.cursor_generation,
reserve,
cutover_timing: HlsLeaseCutoverTiming::from_reserve(fixture.now_ms, 0, reserve.transition_margin, None),
commit_window: HlsTerminalCommitWindow::CutoverDue,
now_ms: fixture.now_ms,
origin_progress_generation: state.origin_control.progress_generation,
media_readiness_generation: state.activity.media_readiness_generation,
last_media_progress_at_ms: state.origin_control.last_media_progress_at_ms,
});
drop(state);
fixture.preparation = preparation.await.unwrap();
let outcome = super::super::terminal_cutover::commit_ready_terminal_bundle(
HlsTerminalCommitContext {
ctx: &fixture.ctx,
session: &fixture.session,
proxy_session_id: &fixture.proxy_session_id,
lease_id: &fixture.lease_id,
preparation: &fixture.preparation,
now_ms,
})
.await,
HlsTerminalResolution::Committed
);
let terminal = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("terminal unavailable lease");
assert_eq!(
terminal.playback_mode,
HlsLeasePlaybackMode::TerminalUnavailable {
decision_generation: fixture.preparation.decision_generation,
reason: expected,
}
);
}
async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture {
let fixture = late_fixture(name).await;
let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone();
config.reverse_proxy =
Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto {
hls_cache: Some(shared::model::HlsCacheConfigDto {
session_idle_timeout: shared::model::Secs::new(60),
..Default::default()
}),
..Default::default()
}));
fixture.ctx.app_config.config.store(Arc::new(config));
fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await;
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert!(fixture
.ctx
.hls_proxy
.activate_access_lease(
&fixture.lease_id,
&fixture.proxy_session_id,
now_ms,
HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let (plan, asset) = prepare(&fixture).await;
assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed);
fixture
}
#[tokio::test]
async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() {
use crate::{
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
HlsMediaActivityCommitOutcome,
};
let fixture = committed_late_activity_fixture("late-active-media").await;
let manager = &fixture.ctx.hls_proxy;
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let before = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms)
.await
.expect("terminal lease");
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
for index in 1_u16..=11 {
let now_ms = started_at_ms + u64::from(index) * 10_000;
let access = manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms)
.await
.expect("custom media remains available");
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await,
HlsMediaActivityCommitOutcome::Committed
);
if index == 6 {
for key in [
HlsLifecycleEventKey::AccessLeaseValidity {
lease_id: fixture.lease_id.clone(),
proxy_session_id: fixture.proxy_session_id.clone(),
},
HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() },
] {
manager
.handle_lifecycle_event(
&fixture.ctx.active_users,
&fixture.ctx.active_provider,
HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 },
now_ms,
)
.await;
}
}
let lease = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("active terminal lease");
assert_eq!(lease.state, HlsAccessLeaseState::Activated);
assert_eq!(lease.last_seen_at_ms, now_ms);
assert_eq!(lease.valid_until_ms, now_ms + 60_000);
assert!(lease.active_until_ms.is_some_and(|until| until > now_ms));
assert_eq!(lease.playback_mode, before.playback_mode);
assert_eq!(lease.playback_cursor, before.playback_cursor);
assert_eq!(lease.admission_generation, before.admission_generation);
assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms);
}
let expired_at_ms = started_at_ms + 170_000;
assert!(manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms)
.await
.is_err());
let expired = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms)
.await
.expect("expired lease");
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000);
}
#[tokio::test]
async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() {
use crate::{
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
HlsMediaActivityCommitOutcome,
};
let fixture = committed_late_activity_fixture("late-idle-media").await;
let manager = &fixture.ctx.hls_proxy;
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
manager
.handle_lifecycle_event(
&fixture.ctx.active_users,
&fixture.ctx.active_provider,
HlsLifecycleEvent {
key: HlsLifecycleEventKey::AccessLeaseActive {
lease_id: fixture.lease_id.clone(),
proxy_session_id: fixture.proxy_session_id.clone(),
},
due_at_ms: started_at_ms + 30_000,
},
started_at_ms + 40_000,
)
.await;
let idle = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("idle lease");
assert_eq!(idle.state, HlsAccessLeaseState::Idle);
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
let mut access = manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000)
.await
.expect("idle custom media");
let original_generation = access.terminal_generation;
access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1));
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
);
let unchanged = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("unchanged idle lease");
assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms);
assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms);
access.terminal_generation = original_generation;
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
HlsMediaActivityCommitOutcome::Committed
);
let active = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("reactivated lease");
assert_eq!(active.state, HlsAccessLeaseState::Activated);
assert_eq!(active.valid_until_ms, started_at_ms + 100_000);
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await,
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
);
let expired = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000)
.await
.expect("expired lease");
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms);
assert_eq!(expired.valid_until_ms, active.valid_until_ms);
}
#[tokio::test]
async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() {
let mut fixture = late_fixture("late-contended-commit").await;
let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms();
fixture.preparation.cutover_timing =
HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None);
let (plan, asset) = prepare(&fixture).await;
let guard = fixture.session.read().await;
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
drop(guard);
let committed = tokio::time::timeout(Duration::from_secs(3), async {
loop {
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(
&fixture.lease_id,
&fixture.proxy_session_id,
fixture.ctx.hls_proxy.terminal_commit_now_ms(),
)
.await
.expect("lease");
if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode {
break committed;
}
tokio::task::yield_now().await;
}
})
.await
.expect("autonomous late custom commit");
assert_eq!(committed, plan);
assert_eq!(
commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()),
HlsTerminalCommitOutcome::AlreadyCommitted
);
}
#[tokio::test]
async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() {
let mut fixture = late_fixture("late-highwater").await;
for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] {
let guard = fixture
.ctx
.hls_proxy
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("publication guard");
let mut manifest = fixture.preparation.manifest_snapshot.clone();
manifest.first_proxy_seq = last_seq;
manifest.last_proxy_seq = last_seq;
manifest.discontinuity_sequence = discontinuity_sequence;
let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment");
segment.proxy_seq = last_seq;
segment.discontinuity_before = discontinuity_before;
assert!(fixture
.ctx
.hls_proxy
.commit_access_lease_manifest_publication(
&fixture.lease_id,
&fixture.proxy_session_id,
guard,
manifest,
fixture.now_ms
)
.await
.is_committed());
}
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("lease");
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot");
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
let (plan, _) = prepare(&fixture).await;
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n"));
assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n"));
}
#[tokio::test]
async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() {
use crate::terminal_commit::{
terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey,
HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision,
};
let fixture = late_fixture("late-bounded-cas").await;
let (plan, asset) = prepare(&fixture).await;
let coordinator = HlsTerminalCommitRetryCoordinator::default();
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let command = |decision| {
let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding");
HlsTerminalCommitCommand {
key: HlsTerminalCommitOwnerKey {
proxy_session_id: fixture.proxy_session_id.clone(),
lease_id: fixture.lease_id.clone(),
lease_issued_at_ms: fixture.preparation.lease_issued_at_ms,
decision_generation: fixture.preparation.decision_generation,
},
session: Arc::clone(&fixture.session),
session_incarnation: fixture
.ctx
.hls_proxy
.sessions()
.session_incarnation(&fixture.session)
.expect("session incarnation"),
preparation: fixture.preparation.clone(),
decision,
media_guard: None,
asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)),
cancellation_epoch,
submission_token,
}
};
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
now_ms
),
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
));
assert!(matches!(
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms),
HlsTerminalCommitSubmissionDecision::Attempt { .. }
));
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
now_ms + 1
),
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
));
assert!(matches!(
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100),
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
));
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::CustomResponse(plan)),
now_ms + terminal_commit_retry_schedule_budget_ms()
),
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
));
now_ms: fixture.now_ms,
},
Arc::clone(&fixture.asset),
fixture.expected_asset,
fixture.bundle_key,
fixture.ready_bundle(),
)
.await;
assert_eq!(outcome, HlsTerminalCommitOutcome::Committed);
assert!(!fixture.session.read().await.live_window_demands.contains_key(&fixture.lease_id));
gc.run_once(fixture.now_ms + 31_000).await.unwrap();
let metadata = manager.segment_cache().metadata(&live_key).await.unwrap().unwrap();
assert_eq!(tokio::fs::read(metadata.path).await.unwrap(), TERMINAL_ASSET_BYTES);
}
backend/hls/src/availability/tests/owner_fixtures.rs
+1 -1
View File
@@ -338,7 +338,7 @@ pub(super) async fn post_refresh_terminal_fixture_with_bundle_state(
&session,
&proxy_session_id,
&lease_id,
if terminal_asset { 22_000 } else { 7_000 },
if terminal_asset { 36_000 } else { 16_000 },
now_ms,
)
.await;
backend/hls/src/availability/tests/pressure_fixtures.rs
-1
View File
@@ -69,7 +69,6 @@ pub(super) fn evaluated_pressure(
cursor: &HlsLeasePlaybackCursor::default(),
ready_timeline: &ready_timeline,
now_ms: 100,
playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget,
origin_path_degraded: true,
recovery_committed: false,
backend/hls/src/availability/tests/recovery.rs
+25 -16
View File
@@ -327,7 +327,7 @@ fn hls_session_recovery_pressure_new_urgent_lease_controls_atomic_commit() {
.expect("urgent pressure");
assert!(after.decision.evaluate_lease_cutovers);
assert_eq!(after.timing_seed.target_duration_ms, 10_000);
assert_eq!(after.timing_seed.target_duration_ms, 8_000);
}
#[test]
@@ -345,7 +345,7 @@ fn hls_session_recovery_pressure_expired_controller_is_excluded_atomically() {
atomic_pressure_policy(),
)
.expect("urgent pressure");
assert_eq!(urgent.timing_seed.target_duration_ms, 10_000);
assert_eq!(urgent.timing_seed.target_duration_ms, 8_000);
let after_expiry = evaluate_and_commit_session_recovery_pressure(
&mut leases,
@@ -403,7 +403,7 @@ async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeas
}
})
.await
.expect("terminal decision before reserve is exhausted")
.expect("terminal decision at reserve exhaustion")
}
#[tokio::test]
@@ -414,7 +414,7 @@ async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
33_000,
fixture.now_ms,
)
.await;
@@ -440,11 +440,11 @@ async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh
assert_eq!(result, HlsTerminalResolution::LiveAllowed);
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
// R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late.
// R=3,000 ms: the independent owner must wait for exhaustion, without another poll.
// Only read the resulting state; do not issue another terminal/manifest request.
let mode = wait_for_terminal_mode(&fixture).await;
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
assert!(current_time_millis() < fixture.now_ms + 5_100);
assert!(fixture.ctx.hls_proxy.now_ms() >= fixture.now_ms + 3_000);
assert!(fixture.session.read().await.origin_refresh.in_flight);
}
@@ -457,7 +457,7 @@ async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() {
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
33_000,
fixture.now_ms,
)
.await;
@@ -505,7 +505,7 @@ async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() {
let mode = wait_for_terminal_mode(&fixture).await;
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
assert!(current_time_millis() < fixture.now_ms + 5_100);
assert!(fixture.ctx.hls_proxy.now_ms() >= fixture.now_ms + 3_000);
assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry");
assert!(fixture.session.read().await.origin_refresh.in_flight);
assert!(tokio::time::timeout(Duration::from_secs(8), refresh)
@@ -523,7 +523,7 @@ async fn recovered_origin_cancels_the_retained_reserve_cutover() {
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
33_000,
fixture.now_ms,
)
.await;
@@ -558,14 +558,14 @@ async fn recovered_origin_cancels_the_retained_reserve_cutover() {
}
#[tokio::test]
async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() {
async fn expired_splice_deadline_keeps_last_live_segment_when_reserve_is_exhausted() {
let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
26_000,
36_000,
fixture.now_ms,
)
.await;
@@ -592,7 +592,7 @@ async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leavin
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("terminal lease");
assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalTail(_)));
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
@@ -604,12 +604,21 @@ async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leavin
.await,
HlsTerminalResolution::Committed
);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
assert!(fixture.session.read().await.has_terminal_tail_protections());
}
#[tokio::test(start_paused = true)]
async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() {
let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
0,
fixture.now_ms,
)
.await;
fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms);
let lease = fixture
.ctx
@@ -633,14 +642,14 @@ async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_lo
}
#[tokio::test]
async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() {
async fn publication_late_origin_keeps_live_base_after_the_splice_deadline() {
let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
26_000,
36_000,
fixture.now_ms,
)
.await;
@@ -668,5 +677,5 @@ async fn publication_late_origin_can_finish_custom_response_after_the_splice_dea
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("terminal lease");
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_)));
}
backend/hls/src/availability/tests/runtime_fixtures.rs
+22
View File
@@ -256,6 +256,15 @@ pub(super) async fn add_active_post_refresh_lease(
)
.await
.is_activated());
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease_id,
36_000,
fixture.now_ms,
)
.await;
}
pub(super) async fn advance_post_refresh_fixture_playback(
@@ -285,6 +294,19 @@ pub(super) async fn advance_post_refresh_fixture_playback(
)
.await
.expect("fixture segment request starts");
assert_eq!(
ctx.hls_proxy
.mark_delivered_media_for_lease(
session,
lease_id,
proxy_session_id,
identity,
Some(token),
playback_at_ms,
)
.await,
crate::manager::HlsMediaActivityCommitOutcome::Committed
);
assert_eq!(
ctx.hls_proxy
.record_access_lease_segment_request_completed_and_mark_media_if_identity_matches(
backend/hls/src/availability/tests/terminal.rs
+2 -2
View File
@@ -39,7 +39,7 @@ fn autonomous_terminal_live_allowed_means_no_cutover_is_required() {
}
#[test]
fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() {
fn live_reserve_wake_is_at_exhaustion_without_an_immediate_margin_loop() {
let now_ms = 1_000;
let mut reserve = terminal_pending_commit_reserve();
reserve.guaranteed_reserve_ms = reserve.guaranteed_reserve_ms.saturating_add(5_000);
@@ -49,7 +49,7 @@ fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() {
let deadline = live_reserve_deadline(now_ms, reserve, cutover_timing).expect("future acquisition wake");
assert_eq!(deadline.next_reevaluation_at_ms, now_ms.saturating_add(5_000));
assert!(deadline.next_reevaluation_at_ms < deadline.latest_safe_terminal_commit_at_ms);
assert_eq!(deadline.next_reevaluation_at_ms, deadline.latest_safe_terminal_commit_at_ms);
}
#[tokio::test]
backend/hls/src/availability/tests/terminal_fixtures.rs
+115 -11
View File
@@ -19,10 +19,8 @@ impl HlsTerminalPendingCommitFixture {
reason: self.expected_asset.reason,
bundle_key: self.bundle_key,
latest_safe_commit_at_ms: self
.preparation
.cutover_timing
.latest_safe_terminal_commit_at
.as_millis_since_epoch(),
.now_ms
.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()),
}
}
@@ -69,19 +67,18 @@ impl HlsTerminalPendingCommitFixture {
pub(super) fn terminal_pending_commit_reserve() -> HlsLeaseReserveSnapshot {
let transition_margin = HlsTransitionMarginMs::from_millis(12_000);
let guaranteed_reserve_ms = transition_margin
.as_millis()
.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis());
let guaranteed_reserve_ms = 0;
HlsLeaseReserveSnapshot {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: guaranteed_reserve_ms,
conservative_playback_position_ms: 0,
playback_position_known: true,
guaranteed_reserve_ms,
initial_hidden_ready_duration_ms: 0,
transition_margin,
key_readiness_valid_until_ms: None,
recovery_required: true,
cutover_required: false,
cutover_required: true,
}
}
@@ -137,6 +134,7 @@ pub(super) async fn install_terminal_pending_ready_base(
.await
.expect("READY terminal-base bytes commit");
let mut session = session.write().await;
session.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure;
let origin_epoch = session.origin_control.origin_epoch;
session.segments.insert(
40,
@@ -214,7 +212,25 @@ pub(super) async fn terminal_pending_commit_fixture_with_base(
name: &str,
base_segment_bytes: &[u8],
) -> HlsTerminalPendingCommitFixture {
let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() });
let ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() });
terminal_pending_commit_fixture_with_ctx(name, base_segment_bytes, ctx).await
}
pub(super) async fn terminal_pending_commit_fixture_with_cache(
name: &str,
path: &std::path::Path,
) -> HlsTerminalPendingCommitFixture {
let mut ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() });
ctx.hls_proxy = Arc::new(crate::HlsProxyManager::with_cache_settings(path, 300));
ctx.attach_lease_user_accounting();
terminal_pending_commit_fixture_with_ctx(name, TERMINAL_ASSET_BYTES, ctx).await
}
async fn terminal_pending_commit_fixture_with_ctx(
name: &str,
base_segment_bytes: &[u8],
hls_ctx: HlsCtx,
) -> HlsTerminalPendingCommitFixture {
let ctx = &hls_ctx;
let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec());
let asset = snapshot_terminal_media_asset(&terminal_buffer).expect("valid terminal asset fixture");
@@ -333,6 +349,10 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail
));
}
#[allow(
clippy::too_many_lines,
reason = "Full reserve exhaustion and owner-failure lifecycle share this terminal fixture."
)]
pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_live_lease(
failure: HlsAvailabilityReevaluationRegistration,
name: &str,
@@ -385,6 +405,65 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
)
.await
.is_activated());
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.unwrap();
let identity = lease.media_identity().unwrap();
let started_at_ms = fixture.now_ms - 100_000;
let token = fixture
.ctx
.hls_proxy
.record_access_lease_segment_request_started_if_identity_matches(
&fixture.lease_id,
&fixture.proxy_session_id,
identity,
40,
started_at_ms,
)
.await
.unwrap();
assert_eq!(
fixture
.ctx
.hls_proxy
.mark_delivered_media_for_lease(
&fixture.session,
&fixture.lease_id,
&fixture.proxy_session_id,
identity,
Some(token),
started_at_ms
)
.await,
crate::manager::HlsMediaActivityCommitOutcome::Committed
);
assert_eq!(
fixture
.ctx
.hls_proxy
.record_access_lease_segment_request_completed_and_mark_media_if_identity_matches(
&fixture.session,
&fixture.lease_id,
&fixture.proxy_session_id,
identity,
token,
started_at_ms
)
.await,
crate::manager::HlsMediaActivityCommitOutcome::Committed
);
assert!(fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.unwrap()
.playback_cursor
.first_segment_started_at_ms
.is_some());
let outcome = commit_post_refresh_terminal_fallback(
fixture.ctx.clone(),
Arc::clone(&fixture.session),
@@ -397,7 +476,32 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
)
.await;
assert_eq!(outcome, HlsPostRefreshFallbackOutcome::TerminalCommitted);
assert!(matches!(
outcome,
HlsPostRefreshFallbackOutcome::TerminalCommitted | HlsPostRefreshFallbackOutcome::PendingOwnerRegistered
));
if outcome == HlsPostRefreshFallbackOutcome::PendingOwnerRegistered {
tokio::time::timeout(Duration::from_secs(4), async {
loop {
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(
&fixture.lease_id,
&fixture.proxy_session_id,
fixture.ctx.hls_proxy.now_ms(),
)
.await
.unwrap();
if lease.playback_mode != HlsLeasePlaybackMode::Live {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("owned terminal decision finishes");
}
let lease = fixture
.ctx
.hls_proxy
@@ -406,6 +510,6 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
.expect("fallback lease remains stored");
assert!(matches!(
lease.playback_mode,
HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. }
HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. }
));
}
backend/hls/src/critical_handoff.rs
+4 -10
View File
@@ -9,7 +9,7 @@ use super::{
HlsTerminalTailCompatibilityInput,
},
HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsCriticalHandoffStateAccess, HlsLeaseReserveInput,
HlsMediaLeaseIdentity, HlsSession, HlsTsTrackSignature, SegmentCacheStatus, HLS_PLAYBACK_RATE_GUARD_MILLI,
HlsMediaLeaseIdentity, HlsSession, HlsTsTrackSignature, SegmentCacheStatus,
};
use log::{debug, warn};
use std::{future::Future, sync::Arc};
@@ -88,10 +88,6 @@ struct HlsCriticalHandoffCandidate {
media_exhaustion_deadline_ms: u64,
}
fn guarded_playback_wall_time_ms(media_time_ms: u64) -> u64 {
media_time_ms.saturating_mul(1_000) / u64::from(HLS_PLAYBACK_RATE_GUARD_MILLI.max(1))
}
pub(super) fn select_critical_handoff_lease(
session: &HlsSession,
leases: &[HlsAccessLease],
@@ -119,7 +115,6 @@ pub(super) fn select_critical_handoff_lease(
cursor: &lease.playback_cursor,
ready_timeline: &ready_timeline,
now_ms,
playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0),
origin_path_degraded: true,
recovery_committed: false,
@@ -131,10 +126,9 @@ pub(super) fn select_critical_handoff_lease(
entry.proxy_file_ext.eq_ignore_ascii_case("ts")
&& matches!(&entry.status, SegmentCacheStatus::Ready { .. })
})?;
let media_exhaustion_deadline_ms =
now_ms.saturating_add(guarded_playback_wall_time_ms(reserve.guaranteed_reserve_ms));
let safe_cutover_deadline_ms = media_exhaustion_deadline_ms
.saturating_sub(guarded_playback_wall_time_ms(reserve.transition_margin.as_millis()));
let media_exhaustion_deadline_ms = now_ms.saturating_add(reserve.guaranteed_reserve_ms);
let safe_cutover_deadline_ms =
media_exhaustion_deadline_ms.saturating_sub(reserve.transition_margin.as_millis());
Some(HlsCriticalHandoffCandidate {
lease: lease.clone(),
snapshot: HlsCriticalHandoffSnapshot {
backend/hls/src/cutover.rs
+28 -12
View File
@@ -37,16 +37,15 @@ pub enum HlsTerminalCutoverDecision {
/// Decides terminal publication from one immutable lease/session snapshot.
///
/// A cutover is authorized only by real media reserve and a current snapshot.
/// Acceptance lifecycle states never stand in for media progress: once the
/// the bounded terminal-acquisition window opens, missing, active, exhausted,
/// and committed episodes all continue to terminal-capability evaluation. A structurally
/// superseded snapshot is retried so reserve can be evaluated again.
/// Publication lateness, request counts,
/// recovery ETA, hard operation timeouts, and generic refresh activity are
/// deliberately absent. Pre-cutover scheduling owns all waits; once the
/// acquisition window opens, neither recovery nor terminal-media preparation
/// may postpone the terminal decision beyond the unchanged safe deadline.
/// Publication requires known reserve exhaustion. Acquisition windows and the
/// transition margin may prepare recovery/media earlier, but never authorize CSV.
/// Missing, active, exhausted and committed acceptance episodes can all reach
/// capability evaluation at exhaustion; only a structurally superseded snapshot
/// requires fresh reserve evidence. An in-flight retry cannot delay this decision.
pub fn evaluate_terminal_cutover(input: &HlsTerminalCutoverInput) -> HlsTerminalCutoverDecision {
if !input.reserve.is_exhausted() {
return HlsTerminalCutoverDecision::NotRequired;
}
match input.commit_window {
HlsTerminalCommitWindow::NotDue => return HlsTerminalCutoverDecision::NotRequired,
HlsTerminalCommitWindow::AcquisitionOpen | HlsTerminalCommitWindow::CutoverDue => {}
@@ -116,6 +115,7 @@ mod tests {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: guaranteed_reserve_ms,
conservative_playback_position_ms: 0,
playback_position_known: true,
guaranteed_reserve_ms,
initial_hidden_ready_duration_ms: 0,
transition_margin,
@@ -127,7 +127,7 @@ mod tests {
fn input(acceptance: HlsManifestAcceptanceEpisodeStatus) -> HlsTerminalCutoverInput {
HlsTerminalCutoverInput {
reserve: degraded_reserve(10_000),
reserve: degraded_reserve(0),
commit_window: HlsTerminalCommitWindow::CutoverDue,
acceptance,
required_terminal_media_key: Some(TERMINAL_KEY),
@@ -143,6 +143,22 @@ mod tests {
}
}
#[test]
fn positive_or_unknown_reserve_never_authorizes_csv_even_during_retries() {
let acquisition = crate::recovery_timing::HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis();
for reserve_ms in [10_000 + acquisition, 10_000, 1] {
for window in [HlsTerminalCommitWindow::AcquisitionOpen, HlsTerminalCommitWindow::CutoverDue] {
let mut candidate = input(HlsManifestAcceptanceEpisodeStatus::InFlight { generation: GENERATION });
candidate.reserve = degraded_reserve(reserve_ms);
candidate.commit_window = window;
assert_eq!(evaluate_terminal_cutover(&candidate), HlsTerminalCutoverDecision::NotRequired);
}
}
let mut unknown = input(exhausted());
unknown.reserve.playback_position_known = false;
assert_eq!(evaluate_terminal_cutover(&unknown), HlsTerminalCutoverDecision::NotRequired);
}
#[test]
fn hls_cutover_policy_full_burst_exhaustion_requires_terminal_capability() {
let mut undecided = input(exhausted());
@@ -250,13 +266,13 @@ mod tests {
}
#[test]
fn hls_cutover_policy_acquisition_window_can_commit_before_the_unchanged_margin() {
fn hls_cutover_policy_acquisition_window_cannot_publish_with_positive_reserve() {
let mut acquisition = input(exhausted());
acquisition.reserve = degraded_reserve(10_001);
acquisition.commit_window = HlsTerminalCommitWindow::AcquisitionOpen;
assert!(!acquisition.reserve.cutover_required);
assert_eq!(evaluate_terminal_cutover(&acquisition), HlsTerminalCutoverDecision::CommitTerminalTail);
assert_eq!(evaluate_terminal_cutover(&acquisition), HlsTerminalCutoverDecision::NotRequired);
}
#[test]
backend/hls/src/gc/policy.rs
+6
View File
@@ -63,6 +63,12 @@ impl ProtectedSet {
if let Some(rendered) = &session.last_rendered_manifest {
protected.key_resource_ids.extend(extract_transient_resource_ids(&rendered.body));
}
// The last committed live segment is the mandatory terminal splice
// anchor. Capacity recovery may release consumed history, never this
// anchor while its lease remains valid.
protected
.segment_proxy_seqs
.extend(session.live_window_demands.values().filter_map(|demand| demand.published.last().copied()));
for (_, protection) in session.terminal_tail_protections() {
protected.segment_proxy_seqs.extend(protection.base_proxy_seqs.iter().copied());
}
backend/hls/src/gc/tests/publication.rs
+8 -4
View File
@@ -223,6 +223,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
for seq in 0..8 {
let token = cursor.record_request_started(seq, 1_000 + seq * 10_000);
let token = cursor.bind_request_timing(token, &session.segments[&seq]);
cursor.record_transfer_started(token, 1_000 + seq * 10_000);
let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000);
}
let manifest = HlsLeaseManifestSnapshot {
@@ -261,7 +262,6 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
cursor: &cursor,
ready_timeline: &timeline,
now_ms: 71_000,
playback_rate_guard_milli: 1_050,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000),
origin_path_degraded: true,
recovery_committed: false,
@@ -269,7 +269,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
(timeline.units.len(), reserve)
};
let (before_units, before) = evaluate(&session);
assert_eq!(before.guaranteed_reserve_ms, 34_500);
assert_eq!(before.guaranteed_reserve_ms, 38_000);
assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0));
// Run the actual metadata-removal function used by garbage collection.
assert!(remove_segment_entry(&mut session, 0).is_some());
@@ -282,7 +282,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
assert_eq!(after, before);
let timing =
HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None);
assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500);
assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 97_000);
// Removing the entire historical prefix must keep exactly the same media clock.
for seq in 1..4 {
assert!(remove_segment_entry(&mut session, seq).is_some());
@@ -290,7 +290,7 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
assert_eq!(evaluate(&session).1, before);
// A real hole in the current READY span must still shorten the reserve.
assert!(remove_segment_entry(&mut session, 7).is_some());
assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500);
assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 26_000);
}
fn install_fast_start(session: &mut HlsSession) {
@@ -510,6 +510,10 @@ async fn obsolete_metadata_is_removed_only_after_publication_and_reader_protecti
published: vec![1],
},
);
assert!(
ProtectedSet::from_session_for_capacity(&state, Some(u64::MAX)).segment_proxy_seqs.contains(&1),
"capacity recovery must retain the lease's last live anchor"
);
state.install_terminal_tail_protection(
lease_id.clone(),
HlsTerminalTailProtection {
backend/hls/src/late_custom_response.rs
+18 -2
View File
@@ -1,5 +1,8 @@
//! Finite custom media published after the live splice deadline has elapsed.
//! Legacy standalone lease-plan shape. Published live leases reject this plan;
//! its constructors exist only for regression tests. Unpublished standalone
//! responses remain owned by `runtime_custom_tail`.
#[cfg(test)]
use crate::{
hls_ctx::HlsCtx,
lease::{HlsAccessLeaseId, HlsTerminalTailPreparation},
@@ -12,7 +15,17 @@ use crate::{
terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration},
HlsAccessLeaseState, ProxySessionId,
};
use std::{fmt, fmt::Write, sync::Arc};
#[cfg(not(test))]
use crate::{
lease::HlsAccessLeaseId,
runtime_custom_tail::{standalone_custom_asset_fingerprint, HlsStandaloneCustomPlan, HlsStandaloneCustomPlanKey},
terminal_tail::HlsTerminalTailGeneration,
ProxySessionId,
};
use std::fmt;
#[cfg(test)]
use std::{fmt::Write, sync::Arc};
#[cfg(test)]
use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs};
/// Immutable, lease-bound custom playlist that does not consume a live splice base.
@@ -69,11 +82,13 @@ impl PartialEq for HlsLateCustomResponsePlan {
impl Eq for HlsLateCustomResponsePlan {}
#[cfg(test)]
pub(crate) enum HlsLateCustomResponseError {
Superseded,
Unavailable(HlsTerminalTailCompatibility),
}
#[cfg(test)]
pub(crate) async fn prepare_late_custom_response(
ctx: &HlsCtx,
proxy_session_id: &ProxySessionId,
@@ -144,6 +159,7 @@ pub(crate) async fn prepare_late_custom_response(
}
// Keeping the layout separate makes the publication boundary testable without a live origin or cache.
#[cfg(test)]
fn render_late_manifest(
first_proxy_seq: u64,
discontinuity_sequence: u64,
backend/hls/src/lease/lifecycle.rs
+54
View File
@@ -88,6 +88,10 @@ impl HlsAccessLease {
pub(super) fn end_playback(&mut self) {
self.runtime_policy_revocation = None;
self.revision_bindings = None;
if let Some(snapshot) = self.last_manifest_snapshot.as_mut() {
snapshot.startup_revisions = None;
}
let release_generation = match std::mem::replace(&mut self.playback_mode, HlsLeasePlaybackMode::Ended) {
HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.generation),
HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => {
@@ -341,6 +345,31 @@ impl HlsAccessLeaseStore {
.collect()
}
/// A valid idle lease can still be consuming its previously available
/// media. The terminal observer must survive a pause in playlist polls.
pub(crate) fn terminal_playback_snapshots_for_session(
&mut self,
proxy_session_id: &ProxySessionId,
now_ms: u64,
) -> Vec<HlsAccessLease> {
if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() {
return Vec::new();
}
self.by_lease_id
.values()
.filter(|lease| {
lease.proxy_session_id == *proxy_session_id
&& ((lease.state == HlsAccessLeaseState::Activated
&& lease.active_until_ms.is_some_and(|until| until > now_ms))
|| (matches!(lease.state, HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle)
&& lease.playback_cursor.first_segment_started_at_ms.is_some()))
&& lease.playback_mode == HlsLeasePlaybackMode::Live
&& lease.last_manifest_snapshot.is_some()
})
.cloned()
.collect()
}
pub fn record_segment_request_started_if_identity_matches(
&mut self,
lease_id: &HlsAccessLeaseId,
@@ -390,6 +419,31 @@ impl HlsAccessLeaseStore {
Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment))
}
pub fn record_segment_transfer_started_if_identity_matches(
&mut self,
lease_id: &HlsAccessLeaseId,
proxy_session_id: &ProxySessionId,
expected: HlsMediaLeaseIdentity,
token: HlsPlaybackRequestToken,
started_at_ms: u64,
) -> Option<bool> {
if !expected.is_live() || !self.media_identity_is_current(lease_id, proxy_session_id, expected, started_at_ms) {
return None;
}
let mut cursor = self.by_lease_id.get(lease_id)?.playback_cursor.clone();
let previous_generation = cursor.cursor_generation;
if !cursor.record_transfer_started(token, started_at_ms) {
return None;
}
let changed = cursor.cursor_generation != previous_generation;
if changed {
self.availability_evidence_can_advance().ok()?;
self.by_lease_id.get_mut(lease_id)?.playback_cursor = cursor;
self.advance_availability_evidence(proxy_session_id).ok()?;
}
Some(changed)
}
pub fn record_segment_request_completed_if_identity_matches(
&mut self,
lease_id: &HlsAccessLeaseId,
backend/hls/src/lease/terminal.rs
+20 -29
View File
@@ -50,6 +50,9 @@ impl HlsAccessLease {
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
) -> Result<(), HlsTerminalCommitOutcome> {
if !preparation.trigger.is_runtime_policy() && !preparation.reserve.is_exhausted() {
return Err(HlsTerminalCommitOutcome::CutoverNoLongerRequired);
}
match preparation.runtime_policy_revocation.as_ref() {
Some(revocation) => {
self.runtime_policy_tail_commit_preconditions_match(proxy_session_id, preparation, revocation)
@@ -256,6 +259,19 @@ impl HlsAccessLeaseStore {
) -> HlsTerminalCommitOutcome {
if !plan.matches_route(proxy_session_id, lease_id)
|| plan.generation.0 != preparation.decision_generation
|| (!preparation.trigger.is_runtime_policy()
&& (plan.base_manifest.last_proxy_seq != preparation.manifest_snapshot.last_proxy_seq
|| plan
.base_manifest
.visible_segments
.last()
.zip(preparation.manifest_snapshot.visible_segments.last())
.is_none_or(|(actual, expected)| {
actual.proxy_seq != expected.proxy_seq
|| actual.uri != expected.uri
|| actual.duration_ms != expected.duration_ms
|| actual.encryption != expected.encryption
})))
|| plan.base_manifest.snapshot_generation != preparation.manifest_snapshot_generation
{
return HlsTerminalCommitOutcome::SupersededGeneration;
@@ -298,40 +314,15 @@ impl HlsAccessLeaseStore {
proxy_session_id: &ProxySessionId,
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
plan: std::sync::Arc<crate::late_custom_response::HlsLateCustomResponsePlan>,
plan: &crate::late_custom_response::HlsLateCustomResponsePlan,
) -> HlsTerminalCommitOutcome {
if !plan.matches_route(proxy_session_id, lease_id)
|| plan.generation.0 != preparation.decision_generation
|| plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation
|| preparation.trigger.is_runtime_policy()
{
return HlsTerminalCommitOutcome::SupersededGeneration;
}
if self.refresh_access_lease_validity(lease_id, now_ms).is_err() {
return HlsTerminalCommitOutcome::SupersededGeneration;
}
let Some(lease) = self.by_lease_id.get_mut(lease_id) else {
let Some(lease) = self.response_snapshot(lease_id, proxy_session_id, now_ms) else {
return HlsTerminalCommitOutcome::LeaseNoLongerEligible;
};
if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) {
return outcome;
}
if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms)
{
return outcome;
}
if self.availability_evidence_can_advance().is_err() {
if lease.issued_at_ms != preparation.lease_issued_at_ms || !plan.matches_route(proxy_session_id, lease_id) {
return HlsTerminalCommitOutcome::SupersededGeneration;
}
let Some(lease) = self.by_lease_id.get_mut(lease_id) else {
return HlsTerminalCommitOutcome::LeaseNoLongerEligible;
};
lease.admission_generation = preparation.decision_generation;
lease.playback_mode = HlsLeasePlaybackMode::TerminalCustomResponse(plan);
if self.advance_availability_evidence(proxy_session_id).is_err() {
return HlsTerminalCommitOutcome::SupersededGeneration;
}
HlsTerminalCommitOutcome::Committed
HlsTerminalCommitOutcome::BundleIncompatible
}
pub(crate) fn late_custom_segment_access(
backend/hls/src/lease/tests/terminal.rs
+101 -9
View File
@@ -1,12 +1,12 @@
use super::{
build_terminal_tail_plan, lease, manifest_snapshot, publish_manifest_snapshot, snapshot_terminal_media_asset,
timing, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeaseState,
HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeaseCutoverTiming, HlsLeaseManifestPublicationOutcome,
HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode,
HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestAcceptanceGeneration,
HlsPlaybackCompletionOutcome, HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason,
HlsRuntimePolicyRevocationOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection,
HlsTerminalBaseSegmentAvailability, HlsTerminalCommitOutcome, HlsTerminalCommitWindow,
build_terminal_tail_plan, lease, manifest_snapshot, publish_manifest_snapshot as publish_unrouted_manifest,
snapshot_terminal_media_asset, timing, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId,
HlsAccessLeaseState, HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeaseCutoverTiming,
HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSegment,
HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot,
HlsManifestAcceptanceGeneration, HlsPlaybackCompletionOutcome, HlsRuntimeCustomTailAssetIdentity,
HlsRuntimeCustomTailReason, HlsRuntimePolicyRevocationOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState,
HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, HlsTerminalCommitOutcome, HlsTerminalCommitWindow,
HlsTerminalMediaPreparationState, HlsTerminalMediaRequirementOrigin, HlsTerminalTailBuildInput,
HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailPreparationInput,
HlsTransitionMarginMs,
@@ -15,6 +15,19 @@ use crate::ProxySessionId;
use std::sync::Arc;
use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer;
fn publish_manifest_snapshot(
store: &mut HlsAccessLeaseStore,
lease_id: &HlsAccessLeaseId,
session: &ProxySessionId,
mut snapshot: HlsLeaseManifestSnapshot,
now_ms: u64,
) -> HlsLeaseManifestPublicationOutcome {
for segment in Arc::make_mut(&mut snapshot.visible_segments) {
segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", session.0, lease_id.0, segment.proxy_seq).into();
}
publish_unrouted_manifest(store, lease_id, session, snapshot, now_ms)
}
pub(in crate::lease::tests) const TERMINAL_ASSET_BYTES: &[u8] =
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"));
@@ -23,6 +36,7 @@ pub(in crate::lease::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: 12_000,
conservative_playback_position_ms: 12_000,
playback_position_known: true,
guaranteed_reserve_ms: 0,
initial_hidden_ready_duration_ms: 0,
transition_margin: HlsTransitionMarginMs::from_millis(12_000),
@@ -36,6 +50,40 @@ pub(in crate::lease::tests) fn cutover_timing() -> HlsLeaseCutoverTiming {
HlsLeaseCutoverTiming::from_reserve(2_000, 0, HlsTransitionMarginMs::from_millis(12_000), None)
}
#[test]
fn terminal_monitor_keeps_started_idle_leases_until_validity_expires() {
let session = ProxySessionId("idle-terminal-monitor".into());
let lease_id = HlsAccessLeaseId("idle-terminal-monitor".into());
let mut leases = HlsAccessLeaseStore::default();
leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1_000));
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(1), 2_000).is_committed());
assert!(leases
.activate_access_lease(
&lease_id,
&session,
2_000,
crate::HlsAccessLeaseTiming { active_window_ms: 10, valid_window_ms: 60_000 }
)
.is_activated());
let identity = leases.response_snapshot(&lease_id, &session, 2_000).unwrap().media_identity().unwrap();
assert!(
leases.terminal_playback_snapshots_for_session(&session, 2_011).is_empty(),
"unstarted lease after active window"
);
let token =
leases.record_segment_request_started_if_identity_matches(&lease_id, &session, identity, 40, 2_001).unwrap();
assert_eq!(
leases.record_segment_transfer_started_if_identity_matches(&lease_id, &session, identity, token, 2_002),
Some(true)
);
assert_eq!(leases.active_live_playback_snapshots_for_session(&session, 2_011), Vec::new());
leases.by_lease_id.get_mut(&lease_id).unwrap().state = HlsAccessLeaseState::Idle;
let monitored = leases.terminal_playback_snapshots_for_session(&session, 2_011);
assert_eq!(monitored.len(), 1);
assert_eq!(monitored[0].state, HlsAccessLeaseState::Idle);
assert_eq!(leases.terminal_playback_snapshots_for_session(&session, 70_000), Vec::new());
}
pub(in crate::lease::tests) fn manifest_snapshot_for_route(
source_rendered_at_ms: u64,
proxy_session_id: &ProxySessionId,
@@ -87,6 +135,15 @@ pub(in crate::lease::tests) fn terminal_plan_at(
) -> Arc<HlsTerminalTailPlan> {
let mut base_manifest = manifest_snapshot_for_route(1, proxy_session_id, lease_id);
base_manifest.snapshot_generation = 1;
terminal_plan_with_manifest(generation, base_manifest, created_at_ms, false)
}
fn terminal_plan_with_manifest(
generation: u64,
base_manifest: HlsLeaseManifestSnapshot,
created_at_ms: u64,
unavailable_prefix: bool,
) -> Arc<HlsTerminalTailPlan> {
let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec());
let asset = snapshot_terminal_media_asset(&transport_stream).expect("terminal asset");
let expected_asset =
@@ -96,7 +153,11 @@ pub(in crate::lease::tests) fn terminal_plan_at(
.visible_proxy_seqs()
.map(|proxy_seq| HlsTerminalBaseSegmentAvailability {
proxy_seq,
media_state: HlsTerminalBaseMediaState::Ready,
media_state: if unavailable_prefix && proxy_seq != base_manifest.last_proxy_seq {
HlsTerminalBaseMediaState::NotReady
} else {
HlsTerminalBaseMediaState::Ready
},
required_map_ready: true,
required_key_ready: true,
protection: HlsTerminalBaseProtection::Protectable,
@@ -704,3 +765,34 @@ fn late_segment_completion_after_terminal_transition_does_not_advance_cursor() {
assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None);
assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None);
}
#[test]
fn terminal_suffix_keeps_last_live_identity_when_discontinuity_moves_into_the_sequence() {
let session = ProxySessionId("discontinuity-splice".into());
let id = HlsAccessLeaseId("discontinuity-splice".into());
let mut store = HlsAccessLeaseStore::default();
store.prepare_access_lease(lease(id.clone(), &session.0, 1_000));
let mut manifest = manifest_snapshot(1);
Arc::make_mut(&mut manifest.visible_segments).last_mut().unwrap().discontinuity_before = true;
assert!(publish_manifest_snapshot(&mut store, &id, &session, manifest, 2_000).is_committed());
let preparation = prepare_terminal(&mut store, &id, &session, 4, Some(2_000));
let plan = terminal_plan_with_manifest(
preparation.decision_generation,
preparation.manifest_snapshot.clone(),
3_000,
true,
);
assert_eq!(plan.base_manifest.visible_segments.len(), 1);
assert!(!plan.base_manifest.visible_segments[0].discontinuity_before);
assert_eq!(plan.base_manifest.discontinuity_sequence, preparation.manifest_snapshot.discontinuity_sequence + 1);
let mut wrong = (*plan).clone();
Arc::make_mut(&mut wrong.base_manifest.visible_segments)[0].uri = "/wrong-live.ts".into();
assert_eq!(
store.commit_terminal_tail_if_generation_matches(&id, &session, &preparation, 3_000, Arc::new(wrong)),
HlsTerminalCommitOutcome::SupersededGeneration
);
assert_eq!(
store.commit_terminal_tail_if_generation_matches(&id, &session, &preparation, 3_000, plan),
HlsTerminalCommitOutcome::Committed
);
}
backend/hls/src/lib.rs
+1 -1
View File
@@ -239,7 +239,7 @@ pub mod api {
#[cfg(test)]
pub(crate) use self::timeline::TimelineMapError;
pub(crate) use self::{
availability::{hls_recovery_timing_policy, HlsManifestAcceptanceDirective, HLS_PLAYBACK_RATE_GUARD_MILLI},
availability::{hls_recovery_timing_policy, HlsManifestAcceptanceDirective},
availability_reevaluation::HlsAvailabilityReevaluationRegistration,
backpressure::{classify_hls_backpressure, HlsBackpressureState},
cache::{
backend/hls/src/live_window.rs
+8 -3
View File
@@ -158,11 +158,16 @@ impl HlsSession {
pub(crate) fn record_live_window_publication(
&mut self,
lease_id: &crate::HlsAccessLeaseId,
issued_at_ms: u64,
snapshot: &crate::media_reserve::HlsLeaseManifestSnapshot,
) {
if let Some(demand) = self.live_window_demands.get_mut(lease_id) {
demand.published = snapshot.visible_proxy_seqs().collect();
}
let demand = self.live_window_demands.entry(lease_id.clone()).or_insert_with(|| HlsLiveWindowDemand {
issued_at_ms,
view: HlsLiveWindowView::Entry,
published: Vec::new(),
});
demand.issued_at_ms = issued_at_ms;
demand.published = snapshot.visible_proxy_seqs().collect();
self.longest_rendered_playlist_duration_ms =
self.longest_rendered_playlist_duration_ms.max(snapshot.playlist_duration_ms);
for seq in snapshot.visible_proxy_seqs() {
backend/hls/src/manager/activity.rs
+15 -4
View File
@@ -8,7 +8,7 @@ use std::sync::Arc;
#[derive(Debug, Clone, Copy)]
enum HlsMediaActivityCommitKind {
Access,
Delivered,
Delivered(Option<HlsPlaybackRequestToken>),
LiveSegmentCompletion(HlsPlaybackRequestToken),
}
@@ -89,6 +89,7 @@ impl HlsProxyManager {
lease_id: &HlsAccessLeaseId,
proxy_session_id: &ProxySessionId,
lease_identity: HlsMediaLeaseIdentity,
token: Option<HlsPlaybackRequestToken>,
now_ms: u64,
) -> HlsMediaActivityCommitOutcome {
self.commit_media_activity_if_identity_matches(
@@ -97,7 +98,7 @@ impl HlsProxyManager {
proxy_session_id,
lease_identity,
now_ms,
HlsMediaActivityCommitKind::Delivered,
HlsMediaActivityCommitKind::Delivered(token),
)
.await
}
@@ -243,9 +244,19 @@ impl HlsProxyManager {
};
}
let (current, capacity_protection_released) = match kind {
HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered => {
HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered(None) => {
(leases.media_identity_is_current(lease_id, proxy_session_id, lease_identity, now_ms), false)
}
HlsMediaActivityCommitKind::Delivered(Some(token)) => {
let started = leases.record_segment_transfer_started_if_identity_matches(
lease_id,
proxy_session_id,
lease_identity,
token,
now_ms,
);
(started.is_some(), started == Some(true))
}
HlsMediaActivityCommitKind::LiveSegmentCompletion(token) => {
let completion = leases.record_segment_request_completed_if_identity_matches(
lease_id,
@@ -264,7 +275,7 @@ impl HlsProxyManager {
};
}
session.mark_authorized_media_access(now_ms);
if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered) {
if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered(_)) {
session.activity.last_delivered_media_at_ms = Some(now_ms);
}
HlsMediaActivityCommitAttempt::Completed {
backend/hls/src/manager/lifecycle.rs
+7
View File
@@ -16,6 +16,13 @@ pub(super) fn hls_session_idle_protection_retry_at(due_at_ms: u64, now_ms: u64)
}
impl HlsProxyManager {
pub(crate) async fn terminal_playback_snapshots_for_session(
&self,
proxy_session_id: &ProxySessionId,
now_ms: u64,
) -> Vec<HlsAccessLease> {
self.access_leases.write().await.terminal_playback_snapshots_for_session(proxy_session_id, now_ms)
}
pub fn lifecycle(&self) -> &Arc<HlsLifecycleManager> { &self.lifecycle }
pub async fn active_live_playback_snapshots_for_session(
backend/hls/src/manager/publication.rs
+12 -2
View File
@@ -525,7 +525,12 @@ impl HlsProxyManager {
} else {
published_transient_resource_ids
};
return leases.commit_manifest_publication_with_resources(
let mut session_guard = match session.as_ref() {
Some(session) => Some(session.write().await),
None => None,
};
let published = snapshot.clone();
let outcome = leases.commit_manifest_publication_with_resources(
lease_id,
proxy_session_id,
expected,
@@ -533,6 +538,12 @@ impl HlsProxyManager {
published_transient_resource_ids,
now_ms,
);
if outcome.is_committed() {
if let Some(session) = session_guard.as_mut() {
session.record_live_window_publication(lease_id, lease_issued_at_ms, &published);
}
}
return outcome;
};
let Some(current_session) = self.sessions.hold_current_proxy_session(proxy_session_id).await else {
return HlsLeaseManifestPublicationOutcome::Rejected(
@@ -639,7 +650,6 @@ impl HlsProxyManager {
cursor: &lease.playback_cursor,
ready_timeline: &ready_timeline,
now_ms,
playback_rate_guard_milli: super::super::HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0),
origin_path_degraded: true,
recovery_committed: false,
backend/hls/src/manager/revision_recovery.rs
+1 -1
View File
@@ -230,7 +230,7 @@ impl HlsProxyManager {
.commit_manifest_publication_with_resources(lease_id, session_id, expected, snapshot, resources, now_ms);
if outcome.is_committed() {
if let Some(snapshot) = published {
session.record_live_window_publication(lease_id, &snapshot);
session.record_live_window_publication(lease_id, expected.lease_issued_at_ms(), &snapshot);
}
}
outcome
backend/hls/src/manager/terminal.rs
+31 -16
View File
@@ -64,6 +64,11 @@ fn evaluate_terminal_commit_authorization(
now_ms: u64,
) -> HlsTerminalCommitAuthorization {
let recovery_snapshot = hls_acceptance_recovery_snapshot(session, now_ms);
if !preparation.trigger.is_runtime_policy()
&& (!preparation.reserve.is_exhausted() || !session.origin_control.path_is_degraded_at(now_ms))
{
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired);
}
// Acceptance lifecycle generations describe recovery evidence, not media
// continuity. A later episode must not invalidate a cutover snapshot while
// the media progress, readiness, epoch, and lease generations guarded below
@@ -71,21 +76,10 @@ fn evaluate_terminal_commit_authorization(
if !terminal_media_requirement_is_bound_to_preparation(preparation) {
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration);
}
if let HlsTerminalLeaseDecision::CustomResponse(plan) = decision {
if preparation.trigger.is_runtime_policy()
|| !plan.matches_route(&session.proxy_session_id, lease_id)
|| plan.generation.0 != preparation.decision_generation
|| plan.manifest_snapshot_generation != preparation.manifest_snapshot_generation
{
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration);
}
if !session.origin_control.path_is_degraded_at(now_ms) {
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired);
}
if now_ms < preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch() {
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible);
}
return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false };
if matches!(decision, HlsTerminalLeaseDecision::CustomResponse(_)) {
// An ongoing lease must retain its exact live splice base, even after
// the old deadline; a standalone custom playlist cannot do that.
return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible);
}
let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_))
&& !session.can_install_terminal_tail_protection(lease_id);
@@ -604,6 +598,25 @@ impl HlsProxyManager {
if preparation.trigger.is_runtime_policy() {
return None;
}
if let HlsTerminalLeaseDecision::Tail(plan) = &command.decision {
if preparation.manifest_snapshot.startup_revisions.is_some()
&& plan.base_manifest.startup_revisions.is_none()
{
return Some(HlsTerminalCommitOutcome::BundleIncompatible);
}
if let Some(publication) = &plan.base_manifest.startup_revisions {
let seq = preparation.manifest_snapshot.last_proxy_seq;
let Some(bound) = publication.revisions.get(&seq) else {
return Some(HlsTerminalCommitOutcome::BundleIncompatible);
};
if leases.published_segment_revision(lease_id, seq).as_ref() != Some(bound) {
return Some(HlsTerminalCommitOutcome::SupersededGeneration);
}
if !matches!(bound.revision().state(), crate::SegmentRevisionState::Complete(_)) {
return Some(HlsTerminalCommitOutcome::BundleIncompatible);
}
}
}
if session.origin_control.progress_generation == preparation.origin_progress_generation
&& session.activity.media_readiness_generation == preparation.media_readiness_generation
&& session.origin_control.origin_epoch == preparation.origin_epoch
@@ -667,7 +680,7 @@ impl HlsProxyManager {
proxy_session_id,
preparation,
now_ms,
Arc::clone(plan),
plan,
),
(HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches(
lease_id,
@@ -710,6 +723,7 @@ impl HlsProxyManager {
}
return outcome;
}
session.live_window_demands.remove(lease_id);
Self::finish_terminal_commit(leases, session, proxy_session_id);
HlsTerminalCommitOutcome::Committed
}
@@ -744,6 +758,7 @@ impl HlsProxyManager {
session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection);
return outcome;
}
session.live_window_demands.remove(lease_id);
Self::finish_terminal_commit(leases, session, proxy_session_id);
HlsTerminalCommitOutcome::Committed
}
backend/hls/src/manager/tests.rs
+1 -2
View File
@@ -9,8 +9,7 @@ use super::{
recovery_timing::{
HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming,
HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy,
HlsRecoveryWorkload, HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow,
HlsTerminalMediaPreparationState, HlsTransitionMarginMs,
HlsRecoveryWorkload, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, HlsTransitionMarginMs,
},
runtime_custom_tail::HlsRuntimeCustomTailAssetIdentity,
session::{HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY},
backend/hls/src/manager/tests/support.rs
+11 -8
View File
@@ -2,10 +2,9 @@ use super::{
prepared_terminal_bundle_key, snapshot_terminal_media_asset, HlsAcceptanceEpisodeTiming,
HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, HlsManifestAcceptanceGeneration,
HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsProxyManager, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy,
HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard,
HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome,
HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState,
HlsTerminalTailPlan, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs,
HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, HlsTerminalCommitMediaGuard,
HlsTerminalCommitOutcome, HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow,
HlsTerminalMediaPreparationState, HlsTerminalTailPlan, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs,
};
use crate::{
build_terminal_tail_plan,
@@ -31,19 +30,18 @@ pub(in crate::manager::tests) const TERMINAL_ASSET_BYTES: &[u8] =
pub(in crate::manager::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot {
let transition_margin = HlsTransitionMarginMs::from_millis(12_000);
let guaranteed_reserve_ms = transition_margin
.as_millis()
.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis());
let guaranteed_reserve_ms = 0;
HlsLeaseReserveSnapshot {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: 12_000_u64.saturating_add(guaranteed_reserve_ms),
conservative_playback_position_ms: 12_000,
playback_position_known: true,
guaranteed_reserve_ms,
initial_hidden_ready_duration_ms: 0,
transition_margin,
key_readiness_valid_until_ms: None,
recovery_required: true,
cutover_required: false,
cutover_required: true,
}
}
@@ -163,6 +161,10 @@ pub(in crate::manager::tests) async fn publish_manifest_snapshot(
snapshot: HlsLeaseManifestSnapshot,
now_ms: u64,
) -> bool {
let mut snapshot = snapshot;
for segment in Arc::make_mut(&mut snapshot.visible_segments) {
segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into();
}
let Some(guard) = manager.prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms).await
else {
return false;
@@ -312,6 +314,7 @@ pub(in crate::manager::tests) fn complete_failed_acceptance_episode(
session: &mut super::super::super::HlsSession,
now_ms: u64,
) -> u64 {
session.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure;
let generation = begin_test_acceptance_episode(session, now_ms);
let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode");
assert_eq!(episode.generation, generation);
backend/hls/src/manager/tests/terminal.rs
+46 -20
View File
@@ -5,11 +5,11 @@ use super::{
spawn_terminal_commit_retry_worker, terminal_plan, terminal_preparation_request, HlsAccessLeaseDenialMode,
HlsAccessLeaseDenialOutcome, HlsLeaseCutoverTiming, HlsMediaActivityCommitOutcome, HlsProxyManager,
HlsRecoveryExecutionState, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard,
HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitCommand, HlsTerminalCommitMediaGuard,
HlsTerminalCommitOutcome, HlsTerminalCommitOwnerKey, HlsTerminalCommitPayload, HlsTerminalCommitRequest,
HlsTerminalCommitRetryDecision, HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision,
HlsTerminalCommitWindow, HlsTerminalLeaseDecision, HlsTerminalMediaRequirementSource,
HlsTerminalTailPreparationRequest, HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY,
HlsTerminalCommitCommand, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome, HlsTerminalCommitOwnerKey,
HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitRetryDecision,
HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision, HlsTerminalCommitWindow,
HlsTerminalLeaseDecision, HlsTerminalMediaRequirementSource, HlsTerminalTailPreparationRequest,
HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY,
};
use crate::{
media_reserve::HlsLeaseReserveSnapshot, terminal_tail::HlsLeasePlaybackMode, HlsAccessLeaseId, HlsAccessLeaseState,
@@ -155,6 +155,7 @@ async fn hls_terminal_commit_missing_acceptance_exact_ready_bundle_commits_tail(
}
);
session.write().await.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure;
let outcome = commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan);
assert_eq!(outcome, HlsTerminalCommitOutcome::Committed);
@@ -495,7 +496,7 @@ async fn hls_terminal_commit_submission_delayed_stale_incoming_preserves_authori
let HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } = next_terminal_commit_retry(
1,
2_000,
preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(),
2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms(),
) else {
panic!("fixture has retry budget");
};
@@ -606,11 +607,14 @@ async fn hls_terminal_commit_submission_newer_command_replaces_lock_busy_owner()
async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() {
let (manager, session, proxy_session_id, lease_id, preparation, plan) =
prepared_terminal_commit_fixture("retry-deadline").await;
let after_deadline_ms =
preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch().saturating_add(1);
manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms);
let after_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms() + 1;
let session_guard = session.write().await;
assert!(matches!(
commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, Arc::clone(&plan)),
HlsTerminalCommitOutcome::LockBusy { .. }
));
manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms);
assert_eq!(
commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,),
HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed
@@ -630,15 +634,15 @@ async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() {
}
#[tokio::test]
async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_deadline() {
async fn hls_terminal_commit_starts_after_old_deadline_but_retry_budget_is_exclusive() {
let (manager, session, proxy_session_id, lease_id, preparation, plan) =
prepared_terminal_commit_fixture("initial-exclusive-deadline").await;
let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let safe_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms();
manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms);
assert_eq!(
commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,),
HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed
HlsTerminalCommitOutcome::Committed
);
assert!(matches!(
manager
@@ -646,13 +650,13 @@ async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_de
.await
.expect("exclusive deadline leaves the initial-attempt lease live")
.playback_mode,
HlsLeasePlaybackMode::Live
HlsLeasePlaybackMode::TerminalTail(_)
));
assert!(!session.read().await.has_terminal_tail_protections());
assert!(session.read().await.has_terminal_tail_protections());
let (manager, session, proxy_session_id, lease_id, preparation, plan) =
prepared_terminal_commit_fixture("retry-exclusive-deadline").await;
let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let safe_deadline_ms = 2_000 + crate::terminal_commit::terminal_commit_retry_schedule_budget_ms();
let session_guard = session.write().await;
assert!(matches!(
commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,),
@@ -916,7 +920,7 @@ async fn hls_terminal_commit_lease_end_prevents_commit_and_gc_protection() {
complete_failed_acceptance_episode(&mut session, 2_000)
};
let mut preparation_request = terminal_preparation_request(&lease_id, &proxy_session_id, progress_generation);
preparation_request.reserve.guaranteed_reserve_ms = 100_000;
preparation_request.reserve.guaranteed_reserve_ms = 0;
preparation_request.reserve.guaranteed_media_horizon_ms = preparation_request
.reserve
.conservative_playback_position_ms
@@ -1003,6 +1007,7 @@ async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farthe
let far_reserve = HlsLeaseReserveSnapshot {
guaranteed_media_horizon_ms: 36_000,
conservative_playback_position_ms: 12_000,
playback_position_known: true,
guaranteed_reserve_ms: 24_000,
cutover_required: false,
..cutover_reserve()
@@ -1037,10 +1042,7 @@ async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farthe
))
.await
.expect("terminal preparation");
assert_eq!(
preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(),
2_000_u64.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis())
);
assert_eq!(preparation.cutover_timing.lease_exhaustion_at.as_millis_since_epoch(), 2_000);
assert_eq!(far_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 14_000);
let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &terminal_lease_id);
let expected_protection = Arc::clone(&plan.protected_base_proxy_seqs);
@@ -1285,3 +1287,27 @@ async fn unavailable_after_expired_splice_deadline_still_rejects_recovered_origi
HlsLeasePlaybackMode::Live
);
}
#[tokio::test]
async fn final_terminal_authorization_rejects_every_positive_reserve_and_unknown_zero() {
let (manager, session, proxy, lease, preparation, plan) = prepared_terminal_commit_fixture("final-zero-gate").await;
let margin = preparation.reserve.transition_margin.as_millis();
let acquisition = crate::recovery_timing::HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis();
for (reserve_ms, known) in [(margin + acquisition, true), (margin, true), (1, true), (0, false)] {
let mut early = preparation.clone();
early.reserve.guaranteed_reserve_ms = reserve_ms;
early.reserve.playback_position_known = known;
assert_eq!(
commit_terminal_plan(&manager, &session, &lease, &proxy, &early, 2_000, Arc::clone(&plan)),
HlsTerminalCommitOutcome::CutoverNoLongerRequired
);
assert_eq!(
manager.access_lease_response_snapshot(&lease, &proxy, 2_000).await.unwrap().playback_mode,
HlsLeasePlaybackMode::Live
);
}
assert_eq!(
commit_terminal_plan(&manager, &session, &lease, &proxy, &preparation, 2_000, plan),
HlsTerminalCommitOutcome::Committed
);
}
backend/hls/src/manifest_acceptance/tests/episode.rs
+1 -1
View File
@@ -256,7 +256,7 @@ fn hls_recovery_timing_candidate_binding_and_staging_reduce_eta_without_moving_d
episode
.estimated_recovery_completion_at(generation, 5_000)
.map(HlsEstimatedRecoveryCompletionAtMs::as_millis_since_epoch),
staged_eta.map(|eta| 5_000_u64.saturating_add(eta))
staged_eta.map(|eta| 5_000_u64.saturating_add(eta).min(frozen_deadline.as_millis_since_epoch()))
);
assert_eq!(episode.remaining_recovery_workload(HlsManifestAcceptanceGeneration(5)), None);
backend/hls/src/manifest_fetch/episode.rs
+2 -2
View File
@@ -118,7 +118,7 @@ fn acceptance_episode_timing(
.origin_control
.target_duration_snapshot_ms
.or_else(|| session.target_duration.map(|seconds| u64::from(seconds).saturating_mul(1_000)))
.unwrap_or(15_000);
.unwrap_or_else(|| context.acceptance_timing_seed.map_or(15_000, |seed| seed.target_duration_ms));
let seed = context.acceptance_timing_seed.unwrap_or(HlsAcceptanceEpisodeTimingSeed {
target_duration_ms: fallback_target_duration_ms,
transition_margin: HlsTransitionMarginMs::from_millis(fallback_target_duration_ms),
@@ -129,7 +129,7 @@ fn acceptance_episode_timing(
HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput {
started_at_ms,
burst_plan,
target_duration_ms: seed.target_duration_ms,
target_duration_ms: fallback_target_duration_ms,
transition_margin: seed.transition_margin,
workload: seed.workload,
observed_latency: session.origin_control.recovery_samples.latency_snapshot(),
backend/hls/src/media_reserve.rs
+87 -50
View File
@@ -113,6 +113,8 @@ impl HlsLeaseManifestSnapshot {
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct HlsLeasePlaybackCursor {
pub first_requested_proxy_seq: Option<u64>,
pub first_segment_started_at_ms: Option<u64>,
pub first_segment_started_proxy_seq: Option<u64>,
pub first_segment_completed_at_ms: Option<u64>,
pub highest_contiguous_completed_proxy_seq: Option<u64>,
pub last_requested_proxy_seq: Option<u64>,
@@ -121,6 +123,7 @@ pub struct HlsLeasePlaybackCursor {
request_epoch: u64,
highest_requested_proxy_seq: Option<u64>,
first_requested_media_start_ms: Option<u64>,
first_segment_media_start_ms: Option<u64>,
highest_contiguous_completed_media_end_ms: Option<u64>,
out_of_order_completed_proxy_seqs: BTreeMap<u64, Option<u64>>,
}
@@ -134,6 +137,7 @@ pub struct HlsPlaybackRequestToken {
proxy_seq: u64,
request_epoch: u64,
media_end_ms: Option<u64>,
media_start_ms: Option<u64>,
}
/// Result of committing a fully delivered segment to a playback cursor.
@@ -170,6 +174,9 @@ impl HlsLeasePlaybackCursor {
if forward_seek {
self.request_epoch = self.request_epoch.saturating_add(1);
self.first_requested_proxy_seq = Some(proxy_seq);
self.first_segment_started_at_ms = None;
self.first_segment_started_proxy_seq = None;
self.first_segment_media_start_ms = None;
self.first_segment_completed_at_ms = None;
self.highest_contiguous_completed_proxy_seq = None;
self.highest_requested_proxy_seq = Some(proxy_seq);
@@ -188,7 +195,12 @@ impl HlsLeasePlaybackCursor {
);
self.cursor_generation = self.cursor_generation.saturating_add(1);
HlsPlaybackRequestToken { proxy_seq, request_epoch: self.request_epoch, media_end_ms: None }
HlsPlaybackRequestToken {
proxy_seq,
request_epoch: self.request_epoch,
media_end_ms: None,
media_start_ms: None,
}
}
/// Binds playback evidence to the immutable session media clock before cache entries can be collected.
@@ -203,9 +215,25 @@ impl HlsLeasePlaybackCursor {
self.first_requested_media_start_ms.get_or_insert(segment.media_start_ms);
}
token.media_end_ms = Some(segment.media_start_ms.saturating_add(segment.duration_ms));
token.media_start_ms = Some(segment.media_start_ms);
token
}
/// Starts the 1x media clock only when the first request actually yields bytes.
/// A successor or duplicate cannot reset it; tokens from earlier seeks are rejected.
pub fn record_transfer_started(&mut self, token: HlsPlaybackRequestToken, started_at_ms: u64) -> bool {
if token.request_epoch != self.request_epoch || self.first_requested_proxy_seq.is_none() {
return false;
}
if self.first_segment_started_at_ms.is_none() {
self.first_segment_started_at_ms = Some(started_at_ms);
self.first_segment_started_proxy_seq = Some(token.proxy_seq);
self.first_segment_media_start_ms = token.media_start_ms;
self.cursor_generation = self.cursor_generation.saturating_add(1);
}
true
}
/// Commits a token only after the corresponding response body was delivered completely.
pub fn record_request_completed(
&mut self,
@@ -295,6 +323,7 @@ pub struct HlsLeaseReserveSnapshot {
pub availability_basis: HlsLeaseReserveAvailabilityBasis,
pub guaranteed_media_horizon_ms: u64,
pub conservative_playback_position_ms: u64,
pub playback_position_known: bool,
pub guaranteed_reserve_ms: u64,
pub initial_hidden_ready_duration_ms: u64,
pub transition_margin: HlsTransitionMarginMs,
@@ -303,6 +332,11 @@ pub struct HlsLeaseReserveSnapshot {
pub cutover_required: bool,
}
impl HlsLeaseReserveSnapshot {
/// Missing playback evidence is not proof that the player consumed its reserve.
pub fn is_exhausted(self) -> bool { self.playback_position_known && self.guaranteed_reserve_ms == 0 }
}
/// Typed source of guaranteed reserve evidence.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HlsLeaseReserveAvailabilityBasis {
@@ -316,7 +350,6 @@ pub struct HlsLeaseReserveInput<'a> {
pub cursor: &'a HlsLeasePlaybackCursor,
pub ready_timeline: &'a HlsReadyTimelineSnapshot,
pub now_ms: u64,
pub playback_rate_guard_milli: u16,
pub recovery_trigger_budget: HlsRecoveryTriggerBudgetMs,
pub origin_path_degraded: bool,
pub recovery_committed: bool,
@@ -329,6 +362,7 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv
availability_basis: HlsLeaseReserveAvailabilityBasis::TransientOriginBacked,
guaranteed_media_horizon_ms: 0,
conservative_playback_position_ms: 0,
playback_position_known: false,
guaranteed_reserve_ms: 0,
initial_hidden_ready_duration_ms: 0,
transition_margin,
@@ -344,9 +378,18 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv
};
let guaranteed_ready_span = contiguous_ready_span(input.ready_timeline, input.manifest, first_needed_seq);
let visible_tail = aligned_visible_tail_ms(input.ready_timeline, input.manifest);
let guaranteed_media_horizon_ms = guaranteed_ready_span.horizon_ms;
let conservative_playback_position_ms =
conservative_playback_position(&input, visible_tail, guaranteed_media_horizon_ms);
// Fully delivered contiguous history survives metadata GC; incomplete bodies do not advance it.
let guaranteed_media_horizon_ms =
guaranteed_ready_span.horizon_ms.max(input.cursor.highest_contiguous_completed_media_end_ms.unwrap_or(0));
let position = conservative_playback_position(&input);
let playback_position_known = position.is_some();
let conservative_playback_position_ms = position.unwrap_or_else(|| {
input
.cursor
.first_requested_proxy_seq
.and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq))
.map_or(visible_tail, |unit| visible_tail.max(unit.start_ms))
});
let guaranteed_reserve_ms = guaranteed_media_horizon_ms.saturating_sub(conservative_playback_position_ms);
let initial_hidden_ready_duration_ms = guaranteed_media_horizon_ms.saturating_sub(visible_tail);
let transition_margin = HlsTransitionMarginMs::from_millis(
@@ -363,6 +406,7 @@ pub fn evaluate_lease_reserve(input: HlsLeaseReserveInput<'_>) -> HlsLeaseReserv
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms,
conservative_playback_position_ms,
playback_position_known,
guaranteed_reserve_ms,
initial_hidden_ready_duration_ms,
transition_margin,
@@ -461,37 +505,13 @@ fn aligned_visible_tail_ms(timeline: &HlsReadyTimelineSnapshot, manifest: &HlsLe
.map_or(manifest.last_visible_media_end_ms, |unit| unit.start_ms.saturating_add(manifest.playlist_duration_ms))
}
fn conservative_playback_position(
input: &HlsLeaseReserveInput<'_>,
visible_tail_ms: u64,
guaranteed_media_horizon_ms: u64,
) -> u64 {
let Some(first_seq) = input.cursor.first_requested_proxy_seq else {
return visible_tail_ms;
};
let Some(first_start_ms) = input.cursor.first_requested_media_start_ms.or_else(|| {
fn conservative_playback_position(input: &HlsLeaseReserveInput<'_>) -> Option<u64> {
let started_at_ms = input.cursor.first_segment_started_at_ms?;
let first_seq = input.cursor.first_segment_started_proxy_seq?;
let first_start_ms = input.cursor.first_segment_media_start_ms.or_else(|| {
input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == first_seq).map(|unit| unit.start_ms)
}) else {
return visible_tail_ms.max(guaranteed_media_horizon_ms);
};
if input.cursor.first_segment_completed_at_ms.is_none() {
return visible_tail_ms.max(first_start_ms);
}
let delivered_end = input
.cursor
.highest_contiguous_completed_media_end_ms
.or_else(|| {
input
.cursor
.highest_contiguous_completed_proxy_seq
.and_then(|seq| input.ready_timeline.units.iter().find(|unit| unit.proxy_seq == seq))
.map(|unit| unit.start_ms.saturating_add(unit.duration_ms))
})
.unwrap_or(guaranteed_media_horizon_ms);
let elapsed_guarded = input.cursor.first_segment_completed_at_ms.map_or(0, |completed_at_ms| {
input.now_ms.saturating_sub(completed_at_ms).saturating_mul(u64::from(input.playback_rate_guard_milli)) / 1_000
});
delivered_end.min(first_start_ms.saturating_add(elapsed_guarded))
})?;
Some(first_start_ms.saturating_add(input.now_ms.saturating_sub(started_at_ms)))
}
fn guaranteed_transition_margin(
@@ -629,6 +649,7 @@ pub fn evaluate_startup_admission(input: HlsStartupAdmissionInput<'_>) -> HlsSta
#[cfg(test)]
mod tests {
mod playback;
use super::*;
fn manifest() -> HlsLeaseManifestSnapshot {
@@ -714,7 +735,6 @@ mod tests {
cursor,
ready_timeline,
now_ms,
playback_rate_guard_milli: 1_050,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000),
origin_path_degraded: true,
recovery_committed: false,
@@ -874,17 +894,19 @@ mod tests {
fn completed_first_segment_uses_measured_start_position() {
let mut cursor = HlsLeasePlaybackCursor::default();
let request = cursor.record_request_started(1, 1_000);
cursor.record_transfer_started(request, 1_000);
assert_eq!(cursor.record_request_completed(request, 1_000), HlsPlaybackCompletionOutcome::Advanced);
let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 3_000);
assert_eq!(reserve.conservative_playback_position_ms, 6_100);
assert_eq!(reserve.guaranteed_reserve_ms, 20_900);
assert_eq!(reserve.conservative_playback_position_ms, 6_000);
assert_eq!(reserve.guaranteed_reserve_ms, 21_000);
}
#[test]
fn sliding_manifest_keeps_cursor_and_ready_horizon_on_one_timeline() {
let mut cursor = HlsLeasePlaybackCursor::default();
let request = cursor.record_request_started(0, 100);
cursor.record_transfer_started(request, 100);
assert_eq!(cursor.record_request_completed(request, 100), HlsPlaybackCompletionOutcome::Advanced);
let mut current_manifest = manifest();
current_manifest.first_proxy_seq = 2;
@@ -911,12 +933,12 @@ mod tests {
current_manifest.last_visible_media_end_ms = 9_000;
assert_eq!(cursor.ready_timeline_start_proxy_seq(current_manifest.first_proxy_seq), 0);
let reserve = reserve(&current_manifest, &cursor, &ready_timeline(), 2_100);
let reserve = reserve(&current_manifest, &cursor, &ready_timeline(), 2_000);
assert_eq!(reserve.conservative_playback_position_ms, 2_100);
assert_eq!(reserve.conservative_playback_position_ms, 1_900);
assert_eq!(reserve.guaranteed_media_horizon_ms, 27_000);
assert_eq!(reserve.initial_hidden_ready_duration_ms, 9_000);
assert_eq!(reserve.guaranteed_reserve_ms, 24_900);
assert_eq!(reserve.guaranteed_reserve_ms, 25_100);
assert!(!reserve.cutover_required);
}
@@ -940,10 +962,14 @@ mod tests {
let second = cursor.record_request_started(1, 200);
let third = cursor.record_request_started(2, 300);
cursor.record_transfer_started(third, 600);
assert_eq!(cursor.record_request_completed(third, 600), HlsPlaybackCompletionOutcome::Buffered);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None);
cursor.record_transfer_started(first, 700);
assert_eq!(cursor.record_request_completed(first, 700), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, Some(0));
cursor.record_transfer_started(second, 800);
assert_eq!(cursor.record_request_completed(second, 800), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, Some(2));
assert_eq!(cursor.first_segment_completed_at_ms, Some(700));
@@ -955,19 +981,23 @@ mod tests {
let old_request = cursor.record_request_started(0, 100);
let seek_request = cursor.record_request_started(4, 200);
cursor.record_transfer_started(old_request, 300);
assert_eq!(cursor.record_request_completed(old_request, 300), HlsPlaybackCompletionOutcome::StaleRequest);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None);
cursor.record_transfer_started(seek_request, 400);
assert_eq!(cursor.record_request_completed(seek_request, 400), HlsPlaybackCompletionOutcome::Advanced);
let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 1_400);
assert_eq!(reserve.conservative_playback_position_ms, 19_050);
assert_eq!(reserve.guaranteed_reserve_ms, 7_950);
assert_eq!(reserve.conservative_playback_position_ms, 19_000);
assert_eq!(reserve.guaranteed_reserve_ms, 8_000);
}
#[test]
fn regressing_clock_saturates_playback_estimate() {
let mut cursor = HlsLeasePlaybackCursor::default();
let request = cursor.record_request_started(1, 2_000);
cursor.record_transfer_started(request, 2_000);
assert_eq!(cursor.record_request_completed(request, 2_000), HlsPlaybackCompletionOutcome::Advanced);
let reserve = reserve(&manifest(), &cursor, &ready_timeline(), 1_000);
@@ -1309,9 +1339,13 @@ mod tests {
cursor.bind_request_timing(token, &session.segments[&seq])
})
.collect::<Vec<_>>();
cursor.record_transfer_started(tokens[0], 1_000);
cursor.record_transfer_started(tokens[2], 1_100);
assert_eq!(cursor.record_request_completed(tokens[2], 1_100), HlsPlaybackCompletionOutcome::Buffered);
cursor.record_transfer_started(tokens[0], 1_200);
assert_eq!(cursor.record_request_completed(tokens[0], 1_200), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(4_000));
cursor.record_transfer_started(tokens[1], 1_300);
assert_eq!(cursor.record_request_completed(tokens[1], 1_300), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(15_000));
let mut current = manifest();
@@ -1330,8 +1364,8 @@ mod tests {
11_200,
);
assert_eq!(before, after);
assert_eq!(after.conservative_playback_position_ms, 10_500);
assert_eq!(after.guaranteed_reserve_ms, 16_500);
assert_eq!(after.conservative_playback_position_ms, 10_200);
assert_eq!(after.guaranteed_reserve_ms, 16_800);
}
#[test]
@@ -1340,6 +1374,7 @@ mod tests {
let mut cursor = HlsLeasePlaybackCursor::default();
let first = cursor.record_request_started(0, 1_000);
let first = cursor.bind_request_timing(first, &session.segments[&0]);
cursor.record_transfer_started(first, 1_000);
assert_eq!(cursor.record_request_completed(first, 1_000), HlsPlaybackCompletionOutcome::Advanced);
let mut current = manifest();
current.first_proxy_seq = 2;
@@ -1350,13 +1385,13 @@ mod tests {
assert_eq!(cursor.ready_timeline_start_proxy_seq(2), 1);
session.segments.remove(&0);
let timeline = session.ready_timeline_snapshot(1, 2_000);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 25_950);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 26_000);
session.segments.get_mut(&1).expect("undelivered segment").status = crate::SegmentCacheStatus::Discovered;
let timeline = session.ready_timeline_snapshot(1, 2_000);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 2_950);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 3_000);
session.segments.remove(&1);
let timeline = session.ready_timeline_snapshot(1, 2_000);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 0);
assert_eq!(reserve(&current, &cursor, &timeline, 2_000).guaranteed_reserve_ms, 3_000);
}
#[test]
@@ -1367,12 +1402,14 @@ mod tests {
let old = cursor.bind_request_timing(old, &session.segments[&0]);
let seek = cursor.record_request_started(4, 2_000);
let seek = cursor.bind_request_timing(seek, &session.segments[&4]);
cursor.record_transfer_started(old, 2_500);
assert_eq!(cursor.record_request_completed(old, 2_500), HlsPlaybackCompletionOutcome::StaleRequest);
cursor.record_transfer_started(seek, 3_000);
assert_eq!(cursor.record_request_completed(seek, 3_000), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.first_requested_media_start_ms, Some(18_000));
assert_eq!(cursor.highest_contiguous_completed_media_end_ms, Some(27_000));
let reserve = reserve(&manifest(), &cursor, &session.ready_timeline_snapshot(0, 4_000), 4_000);
assert_eq!(reserve.conservative_playback_position_ms, 19_050);
assert_eq!(reserve.guaranteed_reserve_ms, 7_950);
assert_eq!(reserve.conservative_playback_position_ms, 19_000);
assert_eq!(reserve.guaranteed_reserve_ms, 8_000);
}
}
backend/hls/src/media_reserve/tests/playback.rs
+154
View File
@@ -0,0 +1,154 @@
use super::*;
#[test]
fn eof_without_transfer_evidence_does_not_start_the_clock_or_prove_exhaustion() {
let mut cursor = HlsLeasePlaybackCursor::default();
let token = cursor.record_request_started(0, 0);
assert_eq!(cursor.record_request_completed(token, 10_000), HlsPlaybackCompletionOutcome::Advanced);
let snapshot = reserve(&manifest(), &cursor, &ready_timeline(), 100_000);
assert_eq!(cursor.first_segment_started_at_ms, None);
assert!(!snapshot.playback_position_known);
assert!(!snapshot.is_exhausted());
}
#[test]
fn concurrent_requests_anchor_to_the_first_actual_transfer_and_keep_its_media_start() {
let session = variable_duration_session();
let mut cursor = HlsLeasePlaybackCursor::default();
let first = cursor.record_request_started(0, 0);
let first = cursor.bind_request_timing(first, &session.segments[&0]);
let successor = cursor.record_request_started(1, 100);
let successor = cursor.bind_request_timing(successor, &session.segments[&1]);
cursor.record_transfer_started(successor, 2_000);
cursor.record_transfer_started(first, 2_500);
assert_eq!(cursor.first_segment_started_at_ms, Some(2_000));
assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 3_000).conservative_playback_position_ms, 5_000);
}
#[test]
fn first_body_chunk_starts_exactly_one_x_before_eof_and_is_not_renewed() {
let session = variable_duration_session();
let mut cursor = HlsLeasePlaybackCursor::default();
let request = cursor.record_request_started(0, 0);
let request = cursor.bind_request_timing(request, &session.segments[&0]);
let unknown = reserve(&manifest(), &cursor, &ready_timeline(), 2_000);
assert!(!unknown.playback_position_known);
assert!(!unknown.is_exhausted());
assert!(cursor.record_transfer_started(request, 3_000));
let playing = reserve(&manifest(), &cursor, &ready_timeline(), 5_000);
assert_eq!(playing.conservative_playback_position_ms, 2_000);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None);
assert_eq!(cursor.first_segment_completed_at_ms, None);
let retry = cursor.record_request_started(0, 6_000);
cursor.record_transfer_started(retry, 6_500);
let next = cursor.record_request_started(1, 7_000);
cursor.record_transfer_started(next, 7_500);
assert_eq!(cursor.record_request_completed(request, 10_000), HlsPlaybackCompletionOutcome::Advanced);
assert_eq!(cursor.first_segment_started_at_ms, Some(3_000));
assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 11_000).conservative_playback_position_ms, 8_000);
}
#[test]
fn one_hour_of_steady_supply_has_no_session_age_drift_then_exhausts_at_one_x() {
let session = variable_duration_session();
let mut cursor = HlsLeasePlaybackCursor::default();
let first = cursor.record_request_started(0, 0);
let first = cursor.bind_request_timing(first, &session.segments[&0]);
cursor.record_transfer_started(first, 3_000);
let mut current = manifest();
let mut last_reserve = 0;
for tick in 0..=360 {
let media_ms = tick * 10_000;
let now_ms = 3_000 + media_ms;
let timeline = HlsReadyTimelineSnapshot {
units: (tick..tick + 6).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect(),
};
current.first_proxy_seq = tick;
current.last_proxy_seq = tick + 5;
current.playlist_duration_ms = 60_000;
current.last_visible_media_end_ms = media_ms + 60_000;
// Completed history allows GC to remove previous metadata, while the
// original EXTINF clock and body-start anchor remain unchanged.
cursor.highest_contiguous_completed_proxy_seq = tick.checked_sub(1);
cursor.highest_contiguous_completed_media_end_ms = Some(media_ms);
let snapshot = reserve(&current, &cursor, &timeline, now_ms);
assert_eq!(snapshot.conservative_playback_position_ms, media_ms);
assert_eq!(snapshot.guaranteed_reserve_ms, 60_000, "tick {tick}");
last_reserve = snapshot.guaranteed_reserve_ms;
}
assert_eq!(last_reserve, 60_000);
let timeline =
HlsReadyTimelineSnapshot { units: (360..366).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() };
for elapsed_ms in [1, 10_000, 59_999, 60_000, 70_000] {
let snapshot = reserve(&current, &cursor, &timeline, 3_603_000 + elapsed_ms);
assert_eq!(snapshot.guaranteed_reserve_ms, 60_000_u64.saturating_sub(elapsed_ms));
assert_eq!(snapshot.is_exhausted(), elapsed_ms >= 60_000);
}
}
#[test]
fn fourteen_minutes_of_supply_then_a_manifest_failure_and_retry_keep_positive_reserve_live() {
let session = variable_duration_session();
let mut cursor = HlsLeasePlaybackCursor::default();
let first = cursor.record_request_started(0, 0);
let first = cursor.bind_request_timing(first, &session.segments[&0]);
cursor.record_transfer_started(first, 3_000);
let mut current = manifest();
for tick in 0..=84 {
let media_ms = tick * 10_000;
let timeline = HlsReadyTimelineSnapshot {
units: (tick..tick + 3).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect(),
};
current.first_proxy_seq = tick;
current.last_proxy_seq = tick + 2;
current.playlist_duration_ms = 30_000;
current.last_visible_media_end_ms = media_ms + 30_000;
cursor.highest_contiguous_completed_proxy_seq = tick.checked_sub(1);
cursor.highest_contiguous_completed_media_end_ms = Some(media_ms);
assert_eq!(reserve(&current, &cursor, &timeline, 3_000 + media_ms).guaranteed_reserve_ms, 30_000);
}
let unchanged =
HlsReadyTimelineSnapshot { units: (84..87).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() };
// One retryable manifest failure degrades the origin while the last READY
// horizon stays unchanged. The former 5% drift would already consume 42s.
for elapsed_ms in [1_000, 2_000] {
let failed = reserve(&current, &cursor, &unchanged, 843_000 + elapsed_ms);
assert_eq!(failed.guaranteed_reserve_ms, 30_000 - elapsed_ms);
assert!(!failed.is_exhausted());
assert!(!failed.cutover_required);
}
// The successful retry supplies one successor. Recovery neither resets
// the first-body anchor nor creates a session-age reserve penalty.
let recovered =
HlsReadyTimelineSnapshot { units: (84..88).map(|seq| ready_unit(seq, seq * 10_000, 10_000)).collect() };
let healthy = evaluate_lease_reserve(HlsLeaseReserveInput {
manifest: &current,
cursor: &cursor,
ready_timeline: &recovered,
now_ms: 846_000,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000),
origin_path_degraded: false,
recovery_committed: true,
});
assert_eq!(healthy.conservative_playback_position_ms, 843_000);
assert_eq!(healthy.guaranteed_reserve_ms, 37_000);
assert!(!healthy.cutover_required);
assert_eq!(cursor.first_segment_started_at_ms, Some(3_000));
}
#[test]
fn seek_discards_the_start_anchor_and_rejects_old_first_chunks() {
let session = variable_duration_session();
let mut cursor = HlsLeasePlaybackCursor::default();
let old = cursor.record_request_started(0, 0);
let old = cursor.bind_request_timing(old, &session.segments[&0]);
cursor.record_transfer_started(old, 1_000);
let seek = cursor.record_request_started(4, 2_000);
let seek = cursor.bind_request_timing(seek, &session.segments[&4]);
assert_eq!(cursor.first_segment_started_at_ms, None);
assert!(!cursor.record_transfer_started(old, 2_500));
assert!(!reserve(&manifest(), &cursor, &ready_timeline(), 2_500).playback_position_known);
cursor.record_transfer_started(seek, 3_000);
assert_eq!(reserve(&manifest(), &cursor, &ready_timeline(), 5_000).conservative_playback_position_ms, 20_000);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None);
}
backend/hls/src/post_refresh_availability.rs
+9 -13
View File
@@ -7,7 +7,7 @@ use super::{
availability_reevaluation::{HlsAvailabilityReevaluationOwnership, HlsAvailabilityReevaluationRegistration},
hls_ctx::HlsCtx,
media_reserve::HlsLeaseReserveSnapshot,
recovery_timing::{HlsLeaseCutoverTiming, HlsTerminalCommitAcquisitionBudgetMs},
recovery_timing::HlsLeaseCutoverTiming,
refresh::HlsPostRefreshAvailabilityAction,
HlsAccessLeaseId, HlsSessionHandle, ProxySessionId,
};
@@ -137,17 +137,13 @@ pub(super) fn live_reserve_deadline(
reserve: HlsLeaseReserveSnapshot,
cutover_timing: HlsLeaseCutoverTiming,
) -> Option<HlsLiveReserveDeadline> {
let trigger_reserve_ms = reserve
.transition_margin
.as_millis()
.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis());
let next_reevaluation_at_ms =
now_ms.saturating_add(reserve.guaranteed_reserve_ms.saturating_sub(trigger_reserve_ms));
let latest_safe_terminal_commit_at_ms = cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let latest_wake_at_ms = latest_safe_terminal_commit_at_ms.checked_sub(1)?;
if !reserve.playback_position_known || reserve.guaranteed_reserve_ms == 0 {
return None;
}
let exhaustion_at_ms = cutover_timing.lease_exhaustion_at.as_millis_since_epoch();
Some(HlsLiveReserveDeadline {
next_reevaluation_at_ms: next_reevaluation_at_ms.min(latest_wake_at_ms),
latest_safe_terminal_commit_at_ms,
next_reevaluation_at_ms: exhaustion_at_ms.max(now_ms.saturating_add(1)),
latest_safe_terminal_commit_at_ms: exhaustion_at_ms,
})
}
@@ -157,7 +153,7 @@ pub(super) async fn evaluate_active_terminal_leases_for_reevaluation(
proxy_session_id: &ProxySessionId,
now_ms: u64,
) -> HlsPostRefreshTerminalEvaluation {
let leases = ctx.hls_proxy.active_live_playback_snapshots_for_session(proxy_session_id, now_ms).await;
let leases = ctx.hls_proxy.terminal_playback_snapshots_for_session(proxy_session_id, now_ms).await;
if leases.is_empty() {
return HlsPostRefreshTerminalEvaluation::NoLiveLease;
}
@@ -233,7 +229,7 @@ pub(super) async fn evaluate_owner_failure_fallback(
proxy_session_id: &ProxySessionId,
now_ms: u64,
) -> HlsPostRefreshFallbackAggregate {
let leases = ctx.hls_proxy.active_live_playback_snapshots_for_session(proxy_session_id, now_ms).await;
let leases = ctx.hls_proxy.terminal_playback_snapshots_for_session(proxy_session_id, now_ms).await;
let mut aggregate = HlsPostRefreshFallbackAggregate::default();
for lease in &leases {
let HlsDetailedTerminalResolution { resolution, live_reserve_deadline } =
backend/hls/src/recovery_timing.rs
+66 -13
View File
@@ -7,6 +7,11 @@ const HLS_ACCEPTANCE_COMMIT_ETA_MS: u64 = 100;
const HLS_ACCEPTANCE_SCHEDULING_ETA_MS: u64 = 100;
const HLS_MANIFEST_REQUEST_FALLBACK_ETA_MS: u64 = 3_000;
/// Decision budget derived from the session's publication cadence, not an HTTP timeout.
pub const fn recovery_budget_ceiling_ms(target_duration_ms: u64) -> u64 {
target_duration_ms.saturating_add(target_duration_ms.div_ceil(2))
}
/// Converts a hard per-operation limit into a conservative fallback estimate
/// without treating unusually large protection timeouts as expected latency.
pub const fn bounded_manifest_request_eta_ms(operation_timeout_ms: u64) -> u64 {
@@ -76,7 +81,6 @@ impl HlsEstimatedRecoveryCompletionAtMs {
}
impl HlsLeaseExhaustionAtMs {
#[cfg(any(test, feature = "test-support"))]
pub const fn as_millis_since_epoch(self) -> u64 { self.0 }
}
@@ -99,9 +103,8 @@ impl HlsTerminalCommitAcquisitionBudgetMs {
pub const fn as_millis(self) -> u64 { self.0 }
}
/// Lease-local phase of terminal acquisition. `AcquisitionOpen` is not an
/// admission or origin-progress transition; it only permits the bounded final
/// commit to begin before the unchanged product transition margin.
/// Lease-local preparation pressure. Neither `AcquisitionOpen` nor the old
/// transition boundary proves exhaustion or authorizes CSV publication.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HlsTerminalCommitWindow {
NotDue,
@@ -401,6 +404,11 @@ impl HlsAcceptanceEpisodeTiming {
calculated_eta_ms.max(input.observed_latency.p95.map_or(0, HlsRecoveryEtaMs::as_millis));
let initial_recovery_eta_ms =
calculated_eta_ms.max(input.observed_latency.p99.map_or(trigger_budget_ms, HlsRecoveryEtaMs::as_millis));
let ceiling_ms = recovery_budget_ceiling_ms(input.target_duration_ms);
log::debug!("HLS recovery decision budget: session_target_duration_ms={} raw_trigger_ms={trigger_budget_ms} raw_eta_ms={initial_recovery_eta_ms} ceiling_ms={ceiling_ms} effective_trigger_ms={} effective_eta_ms={}",
input.target_duration_ms, trigger_budget_ms.min(ceiling_ms), initial_recovery_eta_ms.min(ceiling_ms));
let trigger_budget_ms = trigger_budget_ms.min(ceiling_ms);
let initial_recovery_eta_ms = initial_recovery_eta_ms.min(ceiling_ms);
let deadline_budget_ms = trigger_budget_ms.max(initial_recovery_eta_ms);
Self {
manifest_operation_timeout: input.policy.manifest_operation_timeout,
@@ -438,11 +446,9 @@ impl HlsAcceptanceEpisodeTiming {
media_eta,
self.scheduling_eta,
));
if workload == self.initial_workload {
workload_eta.max(self.initial_recovery_eta)
} else {
workload_eta
}
let eta =
if workload == self.initial_workload { workload_eta.max(self.initial_recovery_eta) } else { workload_eta };
HlsRecoveryEtaMs::from_millis(eta.as_millis().min(recovery_budget_ceiling_ms(self.target_duration_ms)))
}
pub fn estimated_completion_at(
@@ -451,7 +457,9 @@ impl HlsAcceptanceEpisodeTiming {
workload: HlsRecoveryWorkload,
) -> HlsEstimatedRecoveryCompletionAtMs {
HlsEstimatedRecoveryCompletionAtMs::from_millis_since_epoch(
now_ms.saturating_add(self.remaining_eta(workload).as_millis()),
now_ms
.saturating_add(self.remaining_eta(workload).as_millis())
.min(self.acceptance_deadline.as_millis_since_epoch()),
)
}
}
@@ -604,6 +612,51 @@ mod tests {
})
}
#[test]
fn all_decision_budgets_are_capped_after_latency_and_workload_without_changing_timeouts() {
for target_duration_ms in [1, 999, 10_000, 30_001, u64::MAX] {
let workload = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling();
let input = HlsAcceptanceEpisodeTimingInput {
started_at_ms: 1_000,
burst_plan: HlsManifestRecoveryBurstLevel::Beast.plan(),
target_duration_ms,
transition_margin: HlsTransitionMarginMs::from_millis(target_duration_ms),
workload,
observed_latency: HlsObservedRecoveryLatency {
p95: Some(HlsRecoveryEtaMs::from_millis(u64::MAX)),
p99: Some(HlsRecoveryEtaMs::from_millis(u64::MAX)),
},
required_terminal_media_key: None,
terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None },
policy: HlsRecoveryTimingPolicy::new(
HlsOperationTimeoutMs::from_millis(1_000_000),
HlsOperationTimeoutMs::from_millis(2_000_000),
HlsRecoveryEtaMs::from_millis(1_000_000),
HlsRecoveryEtaMs::from_millis(2_000_000),
),
};
let timing = HlsAcceptanceEpisodeTiming::from_input(&input);
let ceiling = recovery_budget_ceiling_ms(target_duration_ms);
assert_eq!(timing.trigger_budget.as_millis(), ceiling);
assert_eq!(timing.initial_recovery_eta.as_millis(), ceiling);
for remaining in [workload, workload.after_full_burst(), HlsRecoveryWorkload::clear_fetch()] {
assert!(timing.remaining_eta(remaining).as_millis() <= ceiling);
for now_ms in [1_000, 2_000, 50_000] {
assert!(
timing.estimated_completion_at(now_ms, remaining).as_millis_since_epoch()
<= timing.acceptance_deadline.as_millis_since_epoch()
);
}
}
assert_eq!(timing.manifest_operation_timeout, input.policy.manifest_operation_timeout);
assert_eq!(timing.media_operation_timeout, input.policy.media_operation_timeout);
}
let small = timing(HlsRecoveryWorkload::clear_fetch());
assert!(small.trigger_budget.as_millis() < 15_000);
assert_eq!(recovery_budget_ceiling_ms(10_000), 15_000);
assert_eq!(recovery_budget_ceiling_ms(999), 1_499);
}
#[test]
fn hls_recovery_timing_workload_classes_have_distinct_expected_etas() {
let staged = timing(HlsRecoveryWorkload {
@@ -720,7 +773,7 @@ mod tests {
assert_eq!(frozen.observed_latency.p99.map(HlsRecoveryEtaMs::as_millis), Some(14_000));
assert_eq!(frozen.remaining_eta(frozen.initial_workload), frozen.initial_recovery_eta);
assert_ne!(frozen.acceptance_deadline, later.acceptance_deadline);
assert_eq!(frozen.acceptance_deadline, later.acceptance_deadline);
assert!(frozen.initial_recovery_eta.as_millis() < frozen.media_operation_timeout.as_millis());
assert_eq!(bounded_manifest_request_eta_ms(120_000), 3_000);
assert_eq!(bounded_manifest_request_eta_ms(750), 750);
@@ -812,8 +865,8 @@ mod tests {
});
assert_eq!(timing.acceptance_deadline.as_millis_since_epoch(), u64::MAX);
assert_eq!(timing.trigger_budget.as_millis(), u64::MAX);
assert_eq!(timing.initial_recovery_eta.as_millis(), u64::MAX);
assert_eq!(timing.trigger_budget.as_millis(), 15_000);
assert_eq!(timing.initial_recovery_eta.as_millis(), 15_000);
}
#[test]
backend/hls/src/refresh/tests.rs
+2 -3
View File
@@ -28,9 +28,8 @@ use super::{
HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming,
HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEncryptionReadiness, HlsRecoveryEtaMs,
HlsRecoveryMapWorkload, HlsRecoveryMediumReadiness, HlsRecoveryObjectReadiness, HlsRecoverySegmentWorkload,
HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope,
HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState,
HlsTransitionMarginMs,
HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope, HlsTerminalCommitWindow,
HlsTerminalMediaPreparationState, HlsTransitionMarginMs,
},
runtime_custom_tail::HlsRuntimeCustomTailReason,
session_store::HlsSessionIncarnation,
backend/hls/src/refresh/tests/acceptance.rs
-1
View File
@@ -678,7 +678,6 @@ pub(in crate::refresh::tests) async fn assert_critical_handoff_evidence(
cursor: &live_lease.playback_cursor,
ready_timeline: &ready_timeline,
now_ms,
playback_rate_guard_milli: super::super::super::HLS_PLAYBACK_RATE_GUARD_MILLI,
recovery_trigger_budget: super::super::super::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0),
origin_path_degraded: true,
recovery_committed: false,
backend/hls/src/refresh/tests/retry_policy.rs
+6 -7
View File
@@ -14,10 +14,10 @@ use super::{
HlsManifestFetchFailureSignal, HlsManifestHttpResponseEvidence, HlsManifestProgress,
HlsManifestRecoveryUnavailableReason, HlsManifestRejectLogReason, HlsManifestSequenceRelation,
HlsOriginManifestFetchContext, HlsOriginManifestFetchRequest, HlsOriginRefreshTriggerOutcome,
HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsTerminalCommitAcquisitionBudgetMs,
HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalTailPreparationRequest,
HlsTransitionMarginMs, LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass,
OriginRefreshRequest, OriginRefreshState, RetryPolicy,
HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsTerminalCommitPayload,
HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs,
LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, OriginRefreshRequest, OriginRefreshState,
RetryPolicy,
};
use crate::{
media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot},
@@ -927,13 +927,12 @@ async fn hls_cutover_policy_advanced_commit_supersedes_preparation_and_keeps_lea
.is_committed());
let prepared_progress_generation = prepare_failed_acceptance_episode(&session).await;
let transition_margin = HlsTransitionMarginMs::from_millis(4_000);
let guaranteed_reserve_ms = transition_margin
.as_millis()
.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis());
let guaranteed_reserve_ms = 0;
let reserve = HlsLeaseReserveSnapshot {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: 8_000_u64.saturating_add(guaranteed_reserve_ms),
conservative_playback_position_ms: 8_000,
playback_position_known: true,
guaranteed_reserve_ms,
initial_hidden_ready_duration_ms: 0,
transition_margin,
backend/hls/src/resource_fetch.rs
+3 -1
View File
@@ -641,7 +641,9 @@ async fn fetch_hls_origin_resource_response_with_timeout(
let response = transport::send(client, request, timeout, &mut deadline).await?;
if !clients.use_manual_redirects || !response.status().is_redirection() {
let response = classify_hls_origin_resource_response(response, target.byte_range_expectation)?;
return Ok((response, deadline.expect("transport returned a connection deadline")));
let deadline = deadline
.ok_or_else(|| HlsOriginResourceFetchError::Transport("HLS connection deadline unavailable".into()))?;
return Ok((response, deadline));
}
if remaining_redirects == 0 {
return Err(HlsOriginResourceFetchError::Redirect);
backend/hls/src/resource_fetch/transport.rs
+2 -1
View File
@@ -34,7 +34,8 @@ pub(super) async fn send(
}
}
}
let deadline = deadline.expect("successful connection establishes the deadline");
let deadline =
deadline.ok_or_else(|| HlsOriginResourceFetchError::Transport("HLS connection deadline unavailable".into()))?;
tokio::time::timeout_at(deadline.deadline(), response)
.await
.map_err(|_| HlsOriginResourceFetchError::Timeout)?
backend/hls/src/response/activity.rs
+1
View File
@@ -33,6 +33,7 @@ impl HlsMediaActivityMarker {
&marker.lease_id,
&marker.proxy_session_id,
marker.lease_identity,
marker.completed_segment,
current_time_millis(),
)
.await;
backend/hls/src/response/cache.rs
+1 -1
View File
@@ -375,7 +375,7 @@ where
mark_response_as_uncompressed(&mut response);
if object.is_media {
if let Some(marker) = &context.media_activity_marker {
response = marker.confirm_media_response(response);
response = cursor_marker.as_ref().unwrap_or(marker).confirm_media_response(response);
}
}
Ok(response)
backend/hls/src/response/revision.rs
+2 -2
View File
@@ -130,7 +130,7 @@ async fn prepare_hls_revision_response(
let stream = RevisionResponseStream {
inner: super::super::progressive_startup::revision_body(revision, lifetime, body_range.start, body_range.end),
qos_meter: Arc::clone(&context.qos_meter),
completion,
completion: completion.clone(),
observation,
expected_len: body_range.length,
bytes_sent: 0,
@@ -142,7 +142,7 @@ async fn prepare_hls_revision_response(
body_range.apply_headers(&mut response, metadata.as_ref(), context.cache_duration_seconds);
mark_response_as_uncompressed(&mut response);
context.metrics.record_cache_hit();
HlsResourceServeOutcome::Ready(marker.confirm_media_response(response))
HlsResourceServeOutcome::Ready(completion.as_ref().unwrap_or(marker).confirm_media_response(response))
}
fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::Instant {
backend/hls/src/response/tests/lifecycle.rs
+25
View File
@@ -1,5 +1,28 @@
use super::*;
#[tokio::test]
async fn empty_body_chunks_do_not_start_playback_but_the_first_data_chunk_does() {
let fixture = CachedSegmentFixture::new(vec![9; 512]).await;
let marker = fixture
.context
.media_activity_marker
.clone()
.unwrap()
.for_segment_request(fixture.segment_file.proxy_seq, current_time_millis())
.await
.unwrap();
let chunks = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::new()), Ok(Bytes::from_static(b"media"))]);
let response = marker.confirm_media_response(axum::http::Response::new(axum::body::Body::from_stream(chunks)));
let mut body = response.into_body();
assert_eq!(body.frame().await.unwrap().unwrap().into_data().unwrap(), Bytes::new());
assert_eq!(fixture.cursor().await.first_segment_started_at_ms, None);
assert_eq!(body.frame().await.unwrap().unwrap().into_data().unwrap(), Bytes::from_static(b"media"));
let cursor = fixture.cursor().await;
assert!(cursor.first_segment_started_at_ms.is_some());
assert_eq!(cursor.first_segment_completed_at_ms, None);
assert_eq!(cursor.highest_contiguous_completed_proxy_seq, None);
}
#[tokio::test]
async fn progressive_http_drop_errors_without_content_length_or_cursor_completion() -> std::io::Result<()> {
let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await;
@@ -105,6 +128,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() {
assert_eq!(requested.first_requested_proxy_seq, Some(12));
assert_eq!(requested.last_requested_proxy_seq, Some(12));
assert_eq!(requested.highest_contiguous_completed_proxy_seq, None);
assert_eq!(requested.first_segment_started_at_ms, None);
assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some());
let mut body = response.into_body();
@@ -126,6 +150,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() {
assert_eq!(dropped.last_requested_proxy_seq, Some(12));
assert_eq!(dropped.highest_contiguous_completed_proxy_seq, None);
assert_eq!(dropped.first_segment_completed_at_ms, None);
assert!(dropped.first_segment_started_at_ms.is_some());
assert_eq!(fixture.meter.bytes_total(), u64::try_from(first_chunk.len()).expect("first chunk size"));
assert_eq!(fixture.access.active_readers(), 0);
}
backend/hls/src/response/tests/revision_recovery.rs
+2 -2
View File
@@ -774,7 +774,7 @@ async fn origin_generation_and_mapping_changes_never_install_retired_retry_bytes
#[tokio::test]
async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> std::io::Result<()> {
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
let failed = failed_optimistic_revision(&fixture, 6).await?;
let _failed = failed_optimistic_revision(&fixture, 6).await?;
install_completed_retry(&fixture).await?;
let mut leases = fixture.manager.access_leases().write().await;
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
@@ -788,7 +788,7 @@ async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() ->
HlsRevisionRecoveryOutcome::Unchanged
);
assert!(leases.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, expires + 1).is_none());
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed));
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), None);
Ok(())
}
backend/hls/src/terminal_commit.rs
+32 -5
View File
@@ -329,10 +329,11 @@ impl HlsTerminalCommitCommand {
}
fn has_independent_commit_deadline(&self) -> bool {
matches!(
&self.decision,
HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)
)
!self.preparation.trigger.is_runtime_policy()
|| matches!(
&self.decision,
HlsTerminalLeaseDecision::CustomResponse(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)
)
}
fn may_replace(&self, current: &Self) -> bool {
@@ -1357,6 +1358,7 @@ mod tests {
availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline,
guaranteed_media_horizon_ms: deadline_ms.saturating_add(100),
conservative_playback_position_ms: 0,
playback_position_known: true,
guaranteed_reserve_ms: deadline_ms.saturating_add(100),
initial_hidden_ready_duration_ms: 0,
transition_margin,
@@ -1365,7 +1367,9 @@ mod tests {
cutover_required: true,
};
let preparation = HlsTerminalTailPreparation {
trigger: super::super::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve,
trigger: super::super::runtime_custom_tail::HlsFiniteTailTrigger::RuntimePolicy(
HlsRuntimeCustomTailReason::LowPriorityPreempted,
),
runtime_policy_revocation: None,
lease_issued_at_ms: 1,
decision_generation: 1,
@@ -1428,6 +1432,29 @@ mod tests {
}
}
#[test]
fn availability_submission_gets_one_fixed_cas_budget_after_the_old_splice_deadline() {
let coordinator = HlsTerminalCommitRetryCoordinator::default();
let mut command = retry_command("availability-budget", 1, 1, 1);
command.preparation.trigger = crate::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve;
command.preparation.reserve.guaranteed_reserve_ms = 0;
let key = command.key.clone();
let first_deadline = 100 + terminal_commit_retry_schedule_budget_ms();
assert!(matches!(coordinator.submit(command, 100), HlsTerminalCommitSubmissionDecision::Attempt { .. }));
assert_eq!(
coordinator.lock_state().owners.get(&key).map(|owner| owner.effective_deadline_ms),
Some(first_deadline)
);
let mut replay = retry_command("availability-budget", 1_000_000, 2, 1);
replay.preparation.trigger = crate::runtime_custom_tail::HlsFiniteTailTrigger::AvailabilityReserve;
replay.preparation.reserve.guaranteed_reserve_ms = 0;
let _ = coordinator.submit(replay, 200);
assert_eq!(
coordinator.lock_state().owners.get(&key).map(|owner| owner.effective_deadline_ms),
Some(first_deadline)
);
}
fn scheduled_worker_token(
decision: &HlsTerminalCommitRetryScheduleDecision,
) -> super::HlsTerminalCommitWorkerToken {
backend/hls/src/terminal_tail.rs
+1
View File
@@ -106,6 +106,7 @@ struct HlsTerminalBaseSegmentProbe {
media_ready: bool,
required_map_ready: bool,
cache_key: Option<super::SegmentCacheKey>,
revision_metadata: Option<super::CachedSegmentMetadata>,
origin_epoch: Option<u64>,
key_evidence: HlsTerminalBaseKeyEvidence,
}
backend/hls/src/terminal_tail/evidence.rs
+26 -7
View File
@@ -62,7 +62,7 @@ impl HlsTerminalBaseEvidence {
/// Makes the intended guard lifetime explicit at the endpoint commit boundary.
#[cfg(any(test, feature = "test-support"))]
pub fn release(self) { drop(self.read_protection) }
pub fn release(self) { drop(self.read_protection); }
/// Transfers the READY-object reader pins to an autonomous terminal commit.
///
@@ -183,7 +183,16 @@ fn snapshot_terminal_base_probes(
.iter()
.map(|segment| {
let entry = session.segments.get(&segment.proxy_seq);
let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. }));
let revision_metadata = manifest
.startup_revisions
.as_ref()
.and_then(|publication| publication.revisions.get(&segment.proxy_seq))
.and_then(|bound| match bound.revision().state() {
crate::SegmentRevisionState::Complete(metadata) => Some(metadata),
_ => None,
});
let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. }))
&& (manifest.startup_revisions.is_none() || revision_metadata.is_some());
let required_map_ready = entry.is_some_and(|entry| {
entry.map_ref.is_none_or(|map_id| {
session
@@ -215,6 +224,7 @@ fn snapshot_terminal_base_probes(
media_ready,
required_map_ready,
cache_key,
revision_metadata,
origin_epoch: media_ready.then(|| entry.map(|entry| entry.origin_key.origin_epoch)).flatten(),
key_evidence,
}
@@ -223,6 +233,19 @@ fn snapshot_terminal_base_probes(
HlsTerminalBaseEvidencePreparation { probes, read_protection: HlsTerminalBaseReadProtection { accesses } }
}
async fn terminal_probe_metadata(
cache: &HlsSegmentCache,
probe: &HlsTerminalBaseSegmentProbe,
) -> Option<crate::CachedSegmentMetadata> {
if let Some(metadata) = probe.revision_metadata.as_ref() {
return Some(metadata.clone());
}
match probe.cache_key.as_ref() {
Some(key) => cache.metadata(key).await.ok().flatten(),
None => None,
}
}
async fn resolve_terminal_base_probes(
segment_cache: &HlsSegmentCache,
manifest: &HlsLeaseManifestSnapshot,
@@ -235,11 +258,7 @@ async fn resolve_terminal_base_probes(
let mut last_track_preprobe_resolution = None;
let mut track_base = None;
for probe in probes {
let metadata = if let Some(cache_key) = probe.cache_key.as_ref() {
segment_cache.metadata(cache_key).await.ok().flatten()
} else {
None
};
let metadata = terminal_probe_metadata(segment_cache, &probe).await;
let protectable = metadata.is_some();
let (required_key_ready, track_encryption, key_failure) = match probe.key_evidence {
HlsTerminalBaseKeyEvidence::Clear => (true, Some(HlsTerminalTrackEncryption::Clear), None),
docs/src/configuration/hls-cache-state-machine.md
+3 -3
View File
@@ -220,7 +220,7 @@ The manifest commit policy tracks multiple signals:
| :--- | :--- | :--- |
| `PublicationLate` | No publication progress has been observed for `1.5 * TARGETDURATION`. | Lateness evidence only. By itself it neither switches host nor closes admission nor cuts over a lease. |
| `RecoveryRequired` | Degraded origin evidence and lease reserve/ETA show that recovery work must start. | Opens one generation-bound acceptance episode and temporarily closes new admission while existing leases use their remaining READY reserve. |
| `Critical` | At least one live lease has reached its reserve/transition boundary without committed recovery. | Evaluates that lease's safe cross-host handoff or prepared terminal decision before its media exhaustion deadline. |
| `Critical` | At least one live lease has reached its reserve/transition boundary without committed recovery. | Prepares that lease's safe cross-host handoff or terminal media. Availability CSV publication still requires known reserve exhaustion. |
| `NoAdmission` | Operator shorthand for the closed startup-admission gate; it is not a separate stored origin enum. | New leases are not admitted from insufficient READY startup media. Existing leases are evaluated independently and are not terminalized as a group. |
An acceptance episode owns an immutable candidate plan, timing budget, workload binding, generation, and deadline. The
@@ -250,7 +250,7 @@ A segment request validates the access lease and then looks up the segment objec
| Known but not ready | Wait briefly or return `503` with `Retry-After`. |
| Missing from session timeline | Return an unavailable/expired response depending on context. |
| Retryable or hard origin failure | Update origin-path evidence and bounded retry state; keep serving lease-specific READY reserve when safe. |
| Lease reserve reaches the transition boundary | Commit a generation-bound prepared terminal decision only after the complete acceptance budget can no longer recover before the safe deadline. |
| Lease reserve reaches the transition boundary | Prepare and re-evaluate early; publish availability CSV only at known reserve exhaustion while the origin is still degraded. |
For a warm canonical manifest request, terminal `channel_unavailable` is not a redirect. The same manifest URL returns an
immutable HTTP `200` media playlist containing a safe live suffix, a discontinuity, finite lease/generation-bound
@@ -304,7 +304,7 @@ the old location.
| Target duration fallback | 15 seconds | Used before an origin target duration is known. |
| Publication lateness | `1.5 * target_duration` | Observation signal only; never a cutover or terminal deadline. |
| Recovery trigger | Lease READY reserve minus full acceptance ETA and transition margin | Starts recovery while the configured burst can still complete. |
| Terminal cutover | Lease-specific safe splice deadline | Bounds a live-tail splice; a late finite Custom Response uses a separate bounded commit budget. |
| Terminal cutover | Known lease media exhaustion | Starts one bounded terminal CAS/retry budget; the last committed live segment remains before CSV. |
| HLS cache GC interval | 30 seconds | Periodic cleanup cadence for cache objects and stale runtime state. |
| Temporary file retention | 30 seconds | Retention window for temporary HLS cache files. |
| Failed segment retention | 10 seconds | Short retention for failed segment state. |
docs/src/configuration/shared-hls-runtime-flow.md
+34 -26
View File
@@ -190,7 +190,8 @@ origin epoch with a discontinuity when required.
`PublicationLate` begins at `1.5 * TARGETDURATION`, but it is only an observation that the origin is publishing late. The
runtime starts recovery and closes startup admission only when lease-specific READY reserve, full-burst recovery ETA, and
transition margin require it. It cuts over only at the lease-specific safe deadline. Manifest request counts never
transition margin require it. For availability failures, CSV is published only after the lease's known media reserve is
completely consumed and the origin remains degraded. Manifest request counts never
terminalize a lease, and initial strip contributes only the READY media actually available to that lease and its measured
start distance.
@@ -216,16 +217,36 @@ offset_pcr(x) = offset_90khz(x) * 300
```
The measured terminal asset duration determines the timestamp stride and the `#EXTINF` value. The target duration is
retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments. Both the
live-tail splice and the late Custom Response use this same prepared bundle.
retained as the playlist limit; a shorter asset does not introduce a timestamp gap between custom segments.
The terminal tail uses this prepared bundle while preserving the lease's live base.
## Warm terminal-tail response
If recovery cannot commit before one lease's safe cutover deadline, Tuliprox evaluates the prepared bundle against the
lease's safe READY live suffix. MPEG-TS terminal media is permitted only for a compatible TS container, inactive
`EXT-X-MAP`, compatible track layout, and a key state that can safely reset to clear media. A required encryption reset is
announced with `#EXT-X-KEY:METHOD=NONE`; an fMP4 MAP or unsupported key transition fails closed instead of creating a TS
splice.
A live lease starts its playback clock at the first nonempty body chunk of a qualified whole-segment GET (including a
full `206 bytes=0-` response). Its estimated position advances at exactly 1x from that segment's EXTINF media start. EOF
separately proves complete delivery; an aborted or in-flight body does not become completed playback history. Manifest
polls, prefetch and ordinary successor requests do not restart the clock. A valid seek starts a new playback section.
Without a transfer/media-time anchor the position is unknown, which does not prove exhaustion.
Contiguous READY media, completed delivery history and required MAP/key readiness determine the available horizon. This
media reserve is not capped. Recovery trigger and ETA budgets are separately limited to
`ceil(1.5 * session TARGETDURATION)` after workload and P95/P99 estimates. HTTP operation timeouts stay independent.
An acceptance episode retains its original deadline; subsequent ETA evaluations cannot move its completion estimate beyond it.
Recovery and bundle preparation may start early, including inside the transition margin and acquisition window. Positive
reserve never authorizes availability CSV publication. Immediate policy cutovers (for example preemption, maintenance or
account blocking) retain their separate rules. The independent terminal observer wakes at reserve exhaustion or on new
evidence, including while an origin retry is in flight and without another player poll.
At proven exhaustion with a still-degraded origin, Tuliprox evaluates the prepared bundle against the last client
manifest committed to that lease. The safe suffix must include that exact manifest's last live segment, with its URI,
EXTINF and bound revision unchanged. Its metadata, dependencies and revision bytes remain protected during the live
lease; terminal publication atomically transfers protection to the immutable tail. A successor publication or lease end
releases the previous mandatory anchor protection.
MPEG-TS terminal media still requires a compatible TS container, inactive `EXT-X-MAP`, compatible track layout, and a
safe key transition. A required encryption reset is announced with `#EXT-X-KEY:METHOD=NONE`. Missing bases, invalid
assets and incompatible media retain their explicit failure reason.
The canonical manifest URL then returns HTTP `200` without `Location`. Its immutable body contains the safe live tail,
the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment references, and `#EXT-X-ENDLIST`:
@@ -234,24 +255,11 @@ the optional key reset, `#EXT-X-DISCONTINUITY`, twelve finite terminal segment r
/hls/shared/live/<proxy_session_id>/<hls_access_lease_id>/terminal/<generation>/<index>.ts
```
If the safe splice deadline has already elapsed, a valid lease with a previously published manifest instead commits a
finite Custom Response without a live suffix. Its canonical manifest returns HTTP `200` with no redirect, twelve
prepared `channel_unavailable` segments under `/cvs/hls/<hls_access_lease_id>/<asset_fingerprint>/<index>.ts`, a clear-key
reset, a discontinuity and `#EXT-X-ENDLIST`. The target duration is retained; media sequence starts after the highest
segment number previously published to that lease, and discontinuity sequence does not decrease. No live cache base,
origin request or timestamp rewrite is needed to serve these custom segments. Playback may stall before this late
warning becomes visible: the fallback does not promise a seamless transition after the deadline.
The late commit has one bounded retry budget independent of the expired splice deadline. Repeated playlist requests
cannot extend it. Lease incarnation, publication/cursor generation, origin recovery and configured asset revision
are revalidated at commit. The frozen manifest and media remain unchanged on replay and asset reload; access still
requires the valid lease and existing user authorization. Successful authorized GET, HEAD and range requests for the
late custom segments activate or renew that exact lease and refresh session activity using the configured
`session_idle_timeout`; playback does not depend on further manifest polls after `#EXT-X-ENDLIST`. Invalid routes,
unsatisfiable ranges and denied authorization do not renew the lease, and an expired lease cannot be reactivated.
Disabled, missing or incompatible custom media can retain an error response. Preparation failures retain their typed
compatibility reason, including `TargetDurationExceeded` with the measured asset and playlist target durations.
The original compatible live-tail splice continues to apply before its deadline.
The former pre-exhaustion splice deadline does not authorize an earlier commit or a standalone CSV fallback. A published
live lease retains its last live segment even if that deadline has elapsed. The first eligible terminal submission
starts one bounded CAS/retry budget; repeated playlist requests cannot renew it. Lease, manifest/cursor generations,
origin progress, asset identity and bound revisions are revalidated at commit. Recovery or a newer client manifest
supersedes a stale decision. A successful terminal publication stays immutable on replay and asset reload.
Only the lease- and generation-bound terminal route serves the live-tail splice's prepared terminal bytes and finite
full/range responses. Existing
docs/src/configuration/shared-hls-troubleshooting.md
+15 -9
View File
@@ -88,13 +88,19 @@ Fix:
## Symptom: custom video `channel_unavailable`
For the warm Shared HLS path, this is a lease-local terminal decision rather than a failure-counter threshold. The
canonical manifest stays on the same URL and returns an HTTP `200` finite playlist when a compatible terminal tail is
ready. Before the splice deadline, an incompatible MAP/key/container/track state can fail closed with `503`.
After the deadline, a valid published lease can receive a finite Custom Response containing only prepared
`channel_unavailable` media, even when its live cache suffix is gone. The playlist uses `/cvs/hls/` URLs and does not
attempt a late TS splice. The player may stall before displaying it. Disabled, missing or invalid custom media,
expired authorization, and exhausted commit capacity can still prevent a usable response.
For an established live lease, an availability failure may publish `channel_unavailable` only after known reserve is
completely consumed and the origin remains degraded. Preparation and recovery may begin earlier. The canonical manifest
returns HTTP `200` with the last live segment of the lease's previous client manifest, the CSV tail and `ENDLIST`. That
live URI and its bound bytes remain available through GC.
There is no CSV-only fallback after the old splice deadline. Missing transfer evidence does not prove reserve
exhaustion. Disabled, missing or invalid custom media, an unavailable base, incompatible MAP/key/container/track state,
expired authorization or exhausted commit capacity retain a concrete error response.
The playback clock begins at the first nonempty qualified segment body chunk and advances at 1x; EOF separately proves
completion. Session age alone adds no reserve penalty. Recovery/ETA decisions are capped at 1.5 session target
durations, independently of media reserve and HTTP timeouts. Inspect `HLS terminal reserve decision` and
`HLS recovery decision budget` for anchor, position, reserve, snapshot generation and raw/effective budgets.
Possible causes:
@@ -102,7 +108,7 @@ Possible causes:
| :--- | :--- |
| Origin progress | Repeated valid HTTP `200` manifests without host-local media progress, or retryable/hard fetch evidence. |
| Manifest acceptance | `HLS manifest acceptance full burst started`, its derived candidate count, and whether a pinned or staged cross-host candidate commits. |
| Lease reserve | The affected lease reaches its recovery/transition boundary before progress commits. |
| Lease reserve | The affected lease exhausts its known reserve while the origin remains degraded. |
| Terminal preparation | `HLS terminal bundle preparation unavailable` with a typed failed or incompatible reason. |
| Terminal commit | `HLS autonomous terminal owner failed closed` or `HLS terminal commit retry owner failed closed`. |
| Segment/MAP/transient media | Required READY reserve cannot be established, or a required object remains unavailable. |
@@ -273,7 +279,7 @@ Adapt the user and group to your deployment.
| `HLS terminal bundle preparation unavailable` | Prepared media is failed or incompatible; no HTTP request attempts an on-demand TS rewrite. |
| `HLS autonomous terminal owner handed off` | The endpoint handed terminal publication to the bounded autonomous commit retry owner. |
| `HLS terminal commit retry owner completed` | A generation-bound retry reached a terminal completion outcome. |
| `HLS terminal commit retry owner failed closed` | Retry exhausted, missed its safe deadline, lost runtime capacity, or observed incompatible media. |
| `HLS terminal commit retry owner failed closed` | The fixed CAS/retry budget expired, runtime capacity was lost, or media was incompatible. |
| `HLS segment cached` | A segment was fetched and committed to the cache. |
| `HLS session switched to transient passthrough` | The manifest required transient resource handling. |
postman/mocks/hls-window/config.yaml
-6
View File
@@ -1,6 +0,0 @@
version: 1
name: hls-window
interceptRequests: true
scenarios:
- name: default
path: default.js
postman/mocks/hls-window/default.js
-35
View File
@@ -1,35 +0,0 @@
const http = require('http');
// Reuses the existing hls-window fixture. Optional controls advance origin
// metadata while a loopback streaming fixture gates actual TS responses.
http.createServer(async (req, res) => {
if (req.method === 'POST' && req.url === '/configure') {
const config = req.body;
if (!config || !/^http:\/\/127\.0\.0\.1:\d+$/.test(config.segmentOrigin) ||
!Number.isSafeInteger(config.firstSequence) || config.firstSequence < 0) {
res.writeHead(400);
return res.end('invalid loopback fixture configuration');
}
await pm.state.set('hls-window.config', config);
res.writeHead(204);
return res.end();
}
if (req.method === 'GET' && req.url === '/index.m3u8') {
const config = await pm.state.get('hls-window.config');
const first = config ? config.firstSequence : 0;
const segmentOrigin = config ? config.segmentOrigin + '/' : '';
res.writeHead(200, { 'content-type': 'application/vnd.apple.mpegurl' });
return res.end('#EXTM3U\n#EXT-X-TARGETDURATION:4\n' +
`#EXT-X-MEDIA-SEQUENCE:${first}\n` +
Array.from({ length: 6 }, (_, i) => `#EXTINF:4,\n${segmentOrigin}${first + i}.ts\n`).join(''));
}
if (req.method === 'GET' && /^\/[0-5]\.ts$/.test(req.url)) {
res.writeHead(200, { 'content-type': 'video/mp2t', 'content-length': 12 });
return setTimeout(() => {
res.write('segment-');
setTimeout(() => res.end('body'), 350);
}, 30);
}
res.writeHead(501);
res.end('endpoint outside the HLS window fixture contract');
}).listen(process.env.PORT);
postman/simulations/hls-window.sim.yaml
-5
View File
@@ -1,5 +0,0 @@
simulation: hls-window
routing: path
mocks:
- routeKey: origin
path: ../mocks/hls-window/config.yaml
tools/tuliprox-testkit/src/bootstrap.rs
+1 -1
View File
@@ -560,7 +560,7 @@ mod tests {
#[tokio::test]
async fn early_process_exit_does_not_wait_for_the_readiness_deadline() {
let config = BootstrapConfig {
command: "/bin/false".to_owned(),
command: "false".to_owned(),
arguments: Vec::new(),
readiness_url: "http://127.0.0.1:1/ready".to_owned(),
readiness_timeout_millis: 5_000,