mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-30 10:42:29 +01:00
fix (hls): recover shared playback and correct live startup windows (#928)
* fix (hls): recover shared HLS retries and stream the first lease segment * fix (hls): select fast-start live windows from the stripped live edge Plan each lease's three-segment entry after applying strip to the full origin window. Share the plan across rendering, required fetches and publication, then grow activated windows forwards while preserving cached predecessors. Retain current-origin mappings during cache eviction and retire obsolete, unprotected metadata during GC. Add window-selection, lease-boundary and metadata-lifecycle regression tests and document the live entry behavior. Validation: 1,238 HLS tests passed (2 ignored); 5 manifest API tests passed; the independent 100-window metadata probe passed. * fix (hls): preserve startup fallback and scope metadata cleanup * fix (hls): wake fast-start fetches from provider fallback * test (hls): cover unpublished fallback during lease recovery
This commit is contained in:
@@ -87,6 +87,7 @@ pub(in crate::api::endpoints::hls_api) async fn prepare_hls_canonical_manifest_o
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
match hls_provider_connections_exhausted_manifest_resolution(
|
||||
app_state,
|
||||
Some(fingerprint),
|
||||
session,
|
||||
&context.username,
|
||||
origin.input,
|
||||
@@ -385,6 +386,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
join_hls_canonical_manifest_owner(
|
||||
HlsCanonicalOwnerHandoffContext {
|
||||
app_state,
|
||||
fingerprint: Some(fingerprint),
|
||||
proxy_session_id: path_proxy_session_id,
|
||||
access_lease_id,
|
||||
expected_lease_issued_at_ms,
|
||||
@@ -431,6 +433,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
if let Some(response) = try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
Some(fingerprint),
|
||||
&session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
@@ -472,6 +475,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
if let Some(response) = try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
Some(fingerprint),
|
||||
&session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
@@ -502,6 +506,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
if let Some(live_response) = try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
Some(fingerprint),
|
||||
&session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
@@ -535,6 +540,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
if let Some(response) = try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
Some(fingerprint),
|
||||
&session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
|
||||
@@ -26,6 +26,108 @@ use tuliprox_hls::api::{
|
||||
ProxySessionId,
|
||||
};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn prepare_live_window_response(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
view: &HlsCachedManifestViewContext<'_>,
|
||||
fingerprint: Option<&crate::model::Fingerprint>,
|
||||
options: HlsCachedManifestOptions,
|
||||
started_at_ms: u64,
|
||||
) -> Option<(HlsMaterializedSharedManifest, HlsLeaseManifestSnapshot, HlsPublishedTransientResourceIds, u64)> {
|
||||
let now_ms = current_time_millis();
|
||||
let lease =
|
||||
app_state.hls.proxy.access_lease_response_snapshot(view.access_lease_id, view.proxy_session_id, now_ms).await?;
|
||||
let (prepared, queued) = {
|
||||
let mut state = session.write().await;
|
||||
state.live_window_strip = app_state.hls.proxy.strip();
|
||||
if !tuliprox_hls::api::hls_live_window_origin_allowed_for_request(&state, options, started_at_ms, now_ms) {
|
||||
return None;
|
||||
}
|
||||
// Keep ordinary forward session commits current when their own window
|
||||
// is ready. A lease's independent entry never depends on this attempt.
|
||||
let _ = state.render_and_store_manifest(now_ms);
|
||||
let desired = state.plan_live_window(tuliprox_hls::api::HlsLiveWindowView::for_lease(&lease))?;
|
||||
if let Some(previous) = lease
|
||||
.last_manifest_snapshot
|
||||
.as_ref()
|
||||
.filter(|previous| desired.visible.first().is_some_and(|first| *first < previous.first_proxy_seq))
|
||||
{
|
||||
if now_ms > previous.delivered_at_ms.saturating_add(previous.playlist_duration_ms) {
|
||||
return None;
|
||||
}
|
||||
let published = previous.startup_revisions.as_ref()?.window_plan.as_ref()?;
|
||||
return Some((
|
||||
HlsMaterializedSharedManifest {
|
||||
body: published.body.to_string(),
|
||||
mode: "normal",
|
||||
initial_strip_outcome: None,
|
||||
},
|
||||
previous.clone(),
|
||||
HlsPublishedTransientResourceIds::default(),
|
||||
now_ms,
|
||||
));
|
||||
}
|
||||
let plan = state.plan_live_window_for_lease(&lease)?;
|
||||
let backpressure = app_state.hls.proxy.segment_worker_pool().classify_backpressure_for_session(&state);
|
||||
let report = state.queue_manifest_fetch_candidates(now_ms, backpressure.allows_prefetch());
|
||||
let queued = report.render_window_queued + report.prefetch_queued > 0;
|
||||
let prepared = tuliprox_hls::api::HlsManifestRenderer::render_live_window(&state, &plan, now_ms).ok().and_then(
|
||||
|rendered| {
|
||||
// An explicit planned view bypasses generic strip. PreserveFullManifest
|
||||
// continues to mean the origin's protected manifest semantics only.
|
||||
let materialized = HlsMaterializedSharedManifest {
|
||||
body: super::materialize_hls_access_manifest(
|
||||
&rendered.body,
|
||||
view.access_lease_id,
|
||||
view.server_path,
|
||||
),
|
||||
mode: "normal",
|
||||
initial_strip_outcome: hls_access_manifest_uses_startup_view(lease.state).then_some(
|
||||
tuliprox_hls::initial_strip::HlsInitialStripOutcome::Applied {
|
||||
mode: match view.strip.mode {
|
||||
shared::model::HlsStripMode::Segments => "segments",
|
||||
shared::model::HlsStripMode::Seconds => "seconds",
|
||||
},
|
||||
configured: view.strip.value,
|
||||
effective: plan.effective_strip_segments,
|
||||
visible_segments: plan.visible.len(),
|
||||
},
|
||||
),
|
||||
};
|
||||
let snapshot = observe_hls_lease_manifest_snapshot_derivation(
|
||||
app_state,
|
||||
view.proxy_session_id,
|
||||
view.access_lease_id,
|
||||
derive_hls_lease_manifest_snapshot(
|
||||
&HlsLeaseManifestSnapshotInput::FastStartWindow {
|
||||
session: &state,
|
||||
plan: &plan,
|
||||
rendered: &rendered,
|
||||
materialized_body: &materialized.body,
|
||||
},
|
||||
now_ms,
|
||||
),
|
||||
)
|
||||
.ok()??;
|
||||
Some((materialized, snapshot, HlsPublishedTransientResourceIds::default(), now_ms))
|
||||
},
|
||||
);
|
||||
(prepared, queued)
|
||||
};
|
||||
if queued {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
let context =
|
||||
super::super::build_hls_segment_fetch_context(app_state, session, None, fingerprint, None).await;
|
||||
let worker = Arc::clone(app_state.hls.proxy.segment_worker_pool());
|
||||
tokio::spawn(async move {
|
||||
worker.wake_scheduler(context, current_time_millis()).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
prepared
|
||||
}
|
||||
|
||||
pub(in crate::api::endpoints::hls_api) struct HlsCachedManifestRead {
|
||||
pub(in crate::api::endpoints::hls_api) transient_body: Option<HlsCachedTransientManifestRead>,
|
||||
pub(in crate::api::endpoints::hls_api) rendered_body: Option<String>,
|
||||
@@ -246,6 +348,7 @@ pub(in crate::api::endpoints::hls_api) fn observe_hls_lease_manifest_snapshot_de
|
||||
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
|
||||
pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response(
|
||||
app_state: &Arc<AppState>,
|
||||
fingerprint: Option<&crate::model::Fingerprint>,
|
||||
session: &HlsSessionHandle,
|
||||
access_lease_id: &HlsAccessLeaseId,
|
||||
access_lease_state: HlsAccessLeaseState,
|
||||
@@ -273,10 +376,27 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
server_path,
|
||||
bandwidth_learning,
|
||||
);
|
||||
let mut last_wait_reason = None;
|
||||
loop {
|
||||
let cached = read_hls_cached_manifest(session, options, started_at_ms).await;
|
||||
if !cached.wait_for_initial_commit {
|
||||
let prepared = if let Some(transient) = cached.transient_body {
|
||||
let fast_start = session.read().await.uses_fast_start_live_window();
|
||||
if cached.wait_for_initial_commit || (cached.rendered_body.is_none() && cached.transient_body.is_none()) {
|
||||
log_hls_manifest_wait_transition(&mut last_wait_reason, "fresh_render_required", session, access_lease_id)
|
||||
.await;
|
||||
}
|
||||
if fast_start || !cached.wait_for_initial_commit {
|
||||
let prepared = if fast_start {
|
||||
let prepared =
|
||||
prepare_live_window_response(app_state, session, &view, fingerprint, options, started_at_ms).await;
|
||||
if prepared.is_none() {
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
return (access_lease_state != HlsAccessLeaseState::Pending)
|
||||
.then(hls_cached_manifest_temporarily_unavailable);
|
||||
}
|
||||
prepared
|
||||
} else if let Some(transient) = cached.transient_body {
|
||||
let materialized = view.materialize(&transient.body, "transient", transient.window_policy);
|
||||
let published_resource_ids = if transient.window_policy.preserves_full_manifest() {
|
||||
transient.published_resource_ids.clone()
|
||||
@@ -342,6 +462,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
if access_lease_state != HlsAccessLeaseState::Pending {
|
||||
return None;
|
||||
}
|
||||
log_hls_manifest_wait_transition(
|
||||
&mut last_wait_reason,
|
||||
"missing_revision_or_head_readiness",
|
||||
session,
|
||||
access_lease_id,
|
||||
)
|
||||
.await;
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
@@ -356,6 +483,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
&& !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms)
|
||||
.await
|
||||
{
|
||||
log_hls_manifest_wait_transition(
|
||||
&mut last_wait_reason,
|
||||
"admission_rejected",
|
||||
session,
|
||||
access_lease_id,
|
||||
)
|
||||
.await;
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
@@ -375,6 +509,12 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
admission_at_ms,
|
||||
)
|
||||
.await;
|
||||
if fast_start && !outcome.is_committed() {
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
return Some(hls_cached_manifest_temporarily_unavailable());
|
||||
}
|
||||
if let Some(snapshot_generation) = outcome.snapshot_generation() {
|
||||
let published_at_ms = current_time_millis();
|
||||
let first_startup_publication =
|
||||
@@ -426,3 +566,28 @@ pub(in crate::api::endpoints::hls_api) async fn wait_for_hls_startup_evidence(
|
||||
tokio::time::sleep(wait_timeout.saturating_sub(elapsed).min(Duration::from_millis(25))).await;
|
||||
true
|
||||
}
|
||||
|
||||
/// Per-request state prevents a 25-ms polling loop from repeating diagnostics.
|
||||
async fn log_hls_manifest_wait_transition(
|
||||
previous: &mut Option<(&'static str, bool, bool)>,
|
||||
reason: &'static str,
|
||||
session: &HlsSessionHandle,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
) {
|
||||
let session = session.read().await;
|
||||
let degraded = session.origin_control.path_condition.is_degraded();
|
||||
let head = session.publishable_origin_head_proxy_seq.and_then(|seq| session.startup.as_ref()?.revisions.get(&seq));
|
||||
let head_ready = head.is_some_and(|head| head.revision().is_startup_head_ready(session.startup_mode()));
|
||||
let state = (reason, degraded, head_ready);
|
||||
if *previous == Some(state) {
|
||||
return;
|
||||
}
|
||||
*previous = Some(state);
|
||||
debug!(
|
||||
"HLS playlist wait: lease={} proxy_session={} reason={reason} degraded={degraded} head_ready={head_ready} origin_generation={} readiness_generation={} head_revision={:?} head_prefix={}",
|
||||
safe_hls_access_lease_id(lease_id), safe_proxy_session_id(&session.proxy_session_id),
|
||||
session.activity.origin_work_generation, session.activity.media_readiness_generation,
|
||||
head.map(|head| head.revision().id()),
|
||||
head.map_or(0, |head| head.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -166,6 +166,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_canonical_owner_failed(
|
||||
|
||||
pub(in crate::api::endpoints::hls_api) struct HlsCanonicalOwnerHandoffContext<'a> {
|
||||
pub(in crate::api::endpoints::hls_api) app_state: &'a Arc<AppState>,
|
||||
pub(in crate::api::endpoints::hls_api) fingerprint: Option<&'a crate::model::Fingerprint>,
|
||||
pub(in crate::api::endpoints::hls_api) proxy_session_id: &'a ProxySessionId,
|
||||
pub(in crate::api::endpoints::hls_api) access_lease_id: &'a HlsAccessLeaseId,
|
||||
pub(in crate::api::endpoints::hls_api) expected_lease_issued_at_ms: Option<u64>,
|
||||
@@ -248,6 +249,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han
|
||||
);
|
||||
if let Some(response) = try_hls_cached_manifest_response(
|
||||
context.app_state,
|
||||
context.fingerprint,
|
||||
current_session,
|
||||
context.access_lease_id,
|
||||
lease.state,
|
||||
|
||||
@@ -622,6 +622,7 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h
|
||||
|
||||
pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline_manifest_response(
|
||||
app_state: &Arc<AppState>,
|
||||
fingerprint: Option<&Fingerprint>,
|
||||
session: &HlsSessionHandle,
|
||||
access_lease_id: &HlsAccessLeaseId,
|
||||
access_lease_state: HlsAccessLeaseState,
|
||||
@@ -668,6 +669,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline
|
||||
|
||||
try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
fingerprint,
|
||||
session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
@@ -682,6 +684,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_or_provider_exhausted_response(
|
||||
app_state: &Arc<AppState>,
|
||||
fingerprint: Option<&Fingerprint>,
|
||||
session: &HlsSessionHandle,
|
||||
username: &str,
|
||||
input: &ConfigInput,
|
||||
@@ -715,6 +718,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_or_provi
|
||||
}
|
||||
if let Some(response) = hls_shared_provisioning_timeline_manifest_response(
|
||||
app_state,
|
||||
fingerprint,
|
||||
session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
@@ -757,6 +761,7 @@ pub(in crate::api::endpoints::hls_api) enum HlsProviderExhaustedResolution {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::api::endpoints::hls_api) async fn hls_provider_connections_exhausted_manifest_resolution(
|
||||
app_state: &Arc<AppState>,
|
||||
fingerprint: Option<&Fingerprint>,
|
||||
session: &HlsSessionHandle,
|
||||
username: &str,
|
||||
input: &ConfigInput,
|
||||
@@ -793,6 +798,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_provider_connections_exhaust
|
||||
HlsProviderExhaustedResolution::Response(
|
||||
hls_shared_provisioning_or_provider_exhausted_response(
|
||||
app_state,
|
||||
fingerprint,
|
||||
session,
|
||||
username,
|
||||
input,
|
||||
|
||||
@@ -29,6 +29,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment(
|
||||
fingerprint: Fingerprint,
|
||||
axum::extract::Path(params): axum::extract::Path<HlsProxySegmentPathParams>,
|
||||
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
|
||||
method: Method,
|
||||
headers: HeaderMap,
|
||||
) -> axum::response::Response {
|
||||
let proxy_session_id = ProxySessionId(params.proxy_session_id);
|
||||
@@ -83,6 +84,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment(
|
||||
&access_context,
|
||||
&fingerprint,
|
||||
&headers,
|
||||
&method,
|
||||
segment_file,
|
||||
now_ms,
|
||||
)
|
||||
@@ -162,12 +164,17 @@ pub(in crate::api::endpoints::hls_api) async fn demand_fetch_hls_live_segment(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_arguments,
|
||||
reason = "Retains request method and headers alongside the validated lease context."
|
||||
)]
|
||||
pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_lease(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
access_context: &HlsAccessContext,
|
||||
fingerprint: &Fingerprint,
|
||||
headers: &HeaderMap,
|
||||
method: &Method,
|
||||
segment_file: HlsSegmentFile,
|
||||
now_ms: u64,
|
||||
) -> axum::response::Response {
|
||||
@@ -181,7 +188,9 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_le
|
||||
let Some(lease_identity) = current_lease.media_identity() else {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
};
|
||||
let response_context = hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await;
|
||||
let mut response_context =
|
||||
hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await;
|
||||
response_context.request_method = method.clone();
|
||||
let response = hls_resource_serve_outcome_response(
|
||||
app_state,
|
||||
access_context,
|
||||
|
||||
@@ -91,6 +91,8 @@ mod headers;
|
||||
mod late_custom_response;
|
||||
mod lease_lifecycle;
|
||||
mod manifest_entry_freshness;
|
||||
mod manifest_live_window;
|
||||
mod manifest_revision_recovery;
|
||||
mod manifest_short_window;
|
||||
mod manifest_startup;
|
||||
mod observability;
|
||||
|
||||
@@ -154,6 +154,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_runtime_bandwidth_man
|
||||
|
||||
let response = super::super::try_hls_cached_manifest_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
&access_lease_id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
|
||||
@@ -0,0 +1,583 @@
|
||||
use super::{normal_manifest, response_body, test_app_state_with_hls_proxy, try_test_hls_cached_manifest_response};
|
||||
use crate::api::model::{HlsAccessLeaseId, HlsAccessLeaseState, HlsProxyManager, HlsSessionKey, SegmentCacheStatus};
|
||||
use axum::http::StatusCode;
|
||||
use shared::model::{HlsCacheConfigDto, HlsStartupMode, HlsStripMode};
|
||||
use std::{fmt::Write as _, sync::Arc, time::Duration};
|
||||
use tuliprox_core::{
|
||||
model::{HlsCacheConfig, StripConfig},
|
||||
utils::current_time_millis,
|
||||
};
|
||||
|
||||
fn origin(
|
||||
first: u64,
|
||||
count: usize,
|
||||
durations: &[u64],
|
||||
) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest {
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-MEDIA-SEQUENCE:{first}\n");
|
||||
for index in 0..count {
|
||||
let sequence = first + u64::try_from(index).unwrap();
|
||||
writeln!(body, "#EXTINF:{},\nhttp://origin.invalid/{sequence}.ts", durations[index % durations.len()]).unwrap();
|
||||
}
|
||||
normal_manifest(&body)
|
||||
}
|
||||
|
||||
fn prepare_revisions(session: &mut tuliprox_hls::api::HlsSession, sequences: &[u64]) {
|
||||
let mode = session.startup_mode();
|
||||
let startup = session.startup.as_mut().unwrap();
|
||||
for seq in sequences {
|
||||
if startup.revisions.contains_key(seq) {
|
||||
continue;
|
||||
}
|
||||
let revision = startup
|
||||
.store
|
||||
.create(
|
||||
session.proxy_session_id.clone(),
|
||||
*seq,
|
||||
if mode == HlsStartupMode::FirstReady {
|
||||
tuliprox_hls::SegmentRevisionKind::Processed
|
||||
} else {
|
||||
tuliprox_hls::SegmentRevisionKind::Raw
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
revision
|
||||
.revision()
|
||||
.complete(tuliprox_hls::api::CachedSegmentMetadata { path: std::path::PathBuf::from("unused"), size: 1 });
|
||||
startup.revisions.insert(*seq, revision);
|
||||
session.segments.get_mut(seq).unwrap().status =
|
||||
SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: current_time_millis() };
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_timeout_preserves_unpublished_fallback_and_existing_lease_retries() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
super::enable_channel_unavailable_custom_response(&app);
|
||||
let session = manager
|
||||
.get_or_create_session(HlsSessionKey::new(1, "unready-window"), b"secret", current_time_millis())
|
||||
.await;
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
for has_origin in [false, true] {
|
||||
if has_origin {
|
||||
session.write().await.apply_origin_manifest(&origin(0, 6, &[4])).unwrap();
|
||||
}
|
||||
for state in [HlsAccessLeaseState::Pending, HlsAccessLeaseState::Activated, HlsAccessLeaseState::Idle] {
|
||||
for wait in [Duration::ZERO, Duration::from_millis(1)] {
|
||||
let id = HlsAccessLeaseId(format!("timeout-{has_origin}-{state:?}-{wait:?}"));
|
||||
super::prepare_pending_test_hls_access_lease(&app, &proxy_id, &id).await;
|
||||
let mut lease =
|
||||
manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap();
|
||||
lease.state = state;
|
||||
if state == HlsAccessLeaseState::Activated {
|
||||
lease.active_until_ms = Some(current_time_millis() + 60_000);
|
||||
}
|
||||
manager.prepare_access_lease(lease).await;
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&id,
|
||||
state,
|
||||
&StripConfig { mode: HlsStripMode::Segments, value: 0 },
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(wait),
|
||||
)
|
||||
.await;
|
||||
if state == HlsAccessLeaseState::Pending {
|
||||
assert!(response.is_none(), "{mode}: unpublished fallback must remain reachable");
|
||||
let fallback =
|
||||
super::super::hls_unpublished_lease_channel_unavailable_response(&app, &proxy_id, &id)
|
||||
.await;
|
||||
assert_eq!(fallback.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(fallback).await.to_vec()).unwrap();
|
||||
assert!(body.contains("#EXT-X-ENDLIST"));
|
||||
assert!(!body.contains("/hls/shared/live/"));
|
||||
super::disable_custom_stream_response(&app);
|
||||
let fallback =
|
||||
super::super::hls_unpublished_lease_channel_unavailable_response(&app, &proxy_id, &id)
|
||||
.await;
|
||||
assert_eq!(fallback.status(), StatusCode::NOT_FOUND);
|
||||
super::enable_channel_unavailable_custom_response(&app);
|
||||
} else {
|
||||
let response = response.expect("existing leases keep the temporary failure response");
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response.headers().contains_key(axum::http::header::RETRY_AFTER));
|
||||
}
|
||||
let lease =
|
||||
manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap();
|
||||
assert!(lease.last_manifest_snapshot.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_provisioning_fallback_wakes_scheduler_for_the_requesting_lease() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
max_segments_prefetch: 0,
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.value = 1;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
super::owner_handoff::enable_hls_provisioning_custom_response(&app);
|
||||
let session = manager
|
||||
.get_or_create_session(HlsSessionKey::new(1, "provisioning-window"), b"secret", current_time_millis())
|
||||
.await;
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 0 };
|
||||
prepare_revisions(&mut state, &[3, 4, 5]);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
let id = HlsAccessLeaseId("provisioning-lease".into());
|
||||
super::prepare_pending_test_hls_access_lease(&app, &proxy_id, &id).await;
|
||||
// Pause before origin I/O: this test verifies the scheduler handoff,
|
||||
// independently of whether a provider slot is available afterwards.
|
||||
let gate = manager.segment_worker_pool().pause_next_origin_prepare_for_test();
|
||||
let response = super::super::hls_shared_provisioning_timeline_manifest_response(
|
||||
&app,
|
||||
Some(&super::test_fingerprint()),
|
||||
&session,
|
||||
&id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: HlsStripMode::Segments, value: 1 },
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(response.is_none(), "the requesting lease still needs its earlier entry segment");
|
||||
tokio::time::timeout(Duration::from_secs(1), gate.entered.notified())
|
||||
.await
|
||||
.expect("the fallback must wake the scheduler for queued live-window work");
|
||||
{
|
||||
let state = session.read().await;
|
||||
assert_eq!(state.segment_prefetch_queue.proxy_seqs(), vec![2]);
|
||||
assert!(matches!(
|
||||
state.segments[&2].status,
|
||||
SegmentCacheStatus::Queued { priority: tuliprox_hls::api::SegmentFetchPriority::RenderWindow, .. }
|
||||
));
|
||||
}
|
||||
assert!(manager.access_leases().write().await.remove_access_lease(&id).is_some());
|
||||
gate.prepare.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(1), gate.prepared.notified()).await.unwrap();
|
||||
gate.proceed.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_api_matrix_has_exact_playlist_snapshot_and_no_second_strip() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
for (count, strip_mode, strip, expected) in [
|
||||
(6, HlsStripMode::Segments, 0, vec![3, 4, 5]),
|
||||
(6, HlsStripMode::Segments, 1, vec![2, 3, 4]),
|
||||
(6, HlsStripMode::Segments, 2, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Segments, 3, vec![0, 1, 2]),
|
||||
(6, HlsStripMode::Segments, u64::MAX, vec![0, 1, 2]),
|
||||
(12, HlsStripMode::Segments, 1, vec![8, 9, 10]),
|
||||
(6, HlsStripMode::Seconds, 0, vec![3, 4, 5]),
|
||||
(6, HlsStripMode::Seconds, 7, vec![2, 3, 4]),
|
||||
(6, HlsStripMode::Seconds, 8, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Seconds, 10, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Seconds, u64::MAX, vec![0, 1, 2]),
|
||||
] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.mode = strip_mode;
|
||||
dto.strip.value = strip;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session =
|
||||
manager.get_or_create_session(HlsSessionKey::new(1, "window"), b"secret", current_time_millis()).await;
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(9_000_000, count, &[4, 4, 4, 4, 3, 7])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: strip_mode, value: strip };
|
||||
prepare_revisions(&mut state, &expected);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let id = HlsAccessLeaseId("entry".into());
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: strip_mode, value: strip },
|
||||
Some("/prefix"),
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK, "{mode} {strip_mode:?} {strip}");
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
let lease = manager
|
||||
.access_lease_response_snapshot(&id, &session.read().await.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.unwrap();
|
||||
let snapshot = lease.last_manifest_snapshot.unwrap();
|
||||
assert_eq!(snapshot.visible_proxy_seqs().collect::<Vec<_>>(), expected);
|
||||
let uris = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::<Vec<_>>();
|
||||
assert_eq!(uris.len(), 3);
|
||||
for (uri, seq) in uris.iter().zip(expected) {
|
||||
assert!(uri.starts_with("/prefix/hls/shared/live/"));
|
||||
assert!(uri.ends_with(&format!("/{seq:06}.ts")));
|
||||
assert!(uri.contains("/entry/"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_entry_does_not_depend_on_the_saved_shared_manifest_prefix() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.value = 1;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session = manager
|
||||
.get_or_create_session(HlsSessionKey::new(1, "saved-prefix"), b"secret", current_time_millis())
|
||||
.await;
|
||||
let saved = {
|
||||
let mut state = session.write().await;
|
||||
let startup = state.startup.take();
|
||||
state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap();
|
||||
for entry in state.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: current_time_millis() };
|
||||
}
|
||||
let saved = state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
assert_eq!(saved.segment_proxy_seqs, vec![0, 1, 2, 3, 4, 5]);
|
||||
state.startup = startup;
|
||||
for entry in state.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Discovered;
|
||||
}
|
||||
prepare_revisions(&mut state, &[2, 3, 4]);
|
||||
saved.body
|
||||
};
|
||||
let id = HlsAccessLeaseId("new-lease".into());
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: HlsStripMode::Segments, value: 1 },
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:2\n"));
|
||||
assert_eq!(body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()).count(), 3);
|
||||
assert_eq!(session.read().await.last_rendered_manifest.as_ref().unwrap().body, saved);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_leases_enter_independently_and_active_windows_only_grow_forward() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.value = 1;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session =
|
||||
manager.get_or_create_session(HlsSessionKey::new(1, "leases"), b"secret", current_time_millis()).await;
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
prepare_revisions(&mut state, &[2, 3, 4]);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
let first = HlsAccessLeaseId("first".into());
|
||||
for (slide, expected) in [
|
||||
(None, vec![2, 3, 4]),
|
||||
(Some(0), vec![2, 3, 4, 5]),
|
||||
(Some(1), vec![2, 3, 4, 5, 6]),
|
||||
(Some(2), vec![2, 3, 4, 5, 6, 7]),
|
||||
(Some(2), vec![2, 3, 4, 5, 6, 7]),
|
||||
] {
|
||||
if let Some(slide) = slide {
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(slide, 6, &[4])).unwrap();
|
||||
prepare_revisions(&mut state, &expected);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&first,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&strip,
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let _ = response_body(response).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let mut lease = manager.access_lease_response_snapshot(&first, &id, current_time_millis()).await.unwrap();
|
||||
assert_eq!(
|
||||
lease.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::<Vec<_>>(),
|
||||
expected
|
||||
);
|
||||
// Preserve this exact lease's bindings and progressive claim state.
|
||||
lease.state = HlsAccessLeaseState::Activated;
|
||||
lease.active_until_ms = Some(current_time_millis() + 60_000);
|
||||
manager.prepare_access_lease(lease).await;
|
||||
}
|
||||
let second = HlsAccessLeaseId("second".into());
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&second,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&strip,
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let _ = response_body(response).await;
|
||||
let id = session.read().await.proxy_session_id.clone();
|
||||
let lease = manager.access_lease_response_snapshot(&second, &id, current_time_millis()).await.unwrap();
|
||||
assert_eq!(lease.last_manifest_snapshot.unwrap().visible_proxy_seqs().collect::<Vec<_>>(), vec![4, 5, 6]);
|
||||
let first = manager.access_lease_response_snapshot(&first, &id, current_time_millis()).await.unwrap();
|
||||
assert_eq!(
|
||||
first.last_manifest_snapshot.unwrap().visible_proxy_seqs().collect::<Vec<_>>(),
|
||||
vec![2, 3, 4, 5, 6, 7]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_active_slides_preserve_duration_without_padding_another_leases_entry() {
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.value = 0;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session = manager
|
||||
.get_or_create_session(HlsSessionKey::new(1, "duration-slides"), b"secret", current_time_millis())
|
||||
.await;
|
||||
let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 };
|
||||
let id = HlsAccessLeaseId("active".into());
|
||||
for (first, durations, expected) in [
|
||||
(100, vec![6, 6, 6, 5], vec![1, 2, 3]),
|
||||
(101, vec![6, 6, 5, 7], vec![1, 2, 3, 4]),
|
||||
(103, vec![5, 7, 6], vec![2, 3, 4, 5]),
|
||||
] {
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(first, durations.len(), &durations)).unwrap();
|
||||
prepare_revisions(&mut state, &expected);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&strip,
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK, "{mode} slide {first}");
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", expected[0])));
|
||||
assert_eq!(body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()).count(), expected.len());
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
let mut lease =
|
||||
manager.access_lease_response_snapshot(&id, &proxy_id, current_time_millis()).await.unwrap();
|
||||
let snapshot = lease.last_manifest_snapshot.as_ref().unwrap();
|
||||
assert_eq!(snapshot.visible_proxy_seqs().collect::<Vec<_>>(), expected);
|
||||
if first != 100 {
|
||||
assert!(snapshot.playlist_duration_ms >= 21_000);
|
||||
}
|
||||
lease.state = HlsAccessLeaseState::Activated;
|
||||
lease.active_until_ms = Some(current_time_millis() + 60_000);
|
||||
manager.prepare_access_lease(lease).await;
|
||||
}
|
||||
let new_id = HlsAccessLeaseId("new".into());
|
||||
// Both the first publication and its active refresh must honor this lease's own head.
|
||||
for _ in 0..2 {
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&new_id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&strip,
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let _ = response_body(response).await;
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
let mut lease =
|
||||
manager.access_lease_response_snapshot(&new_id, &proxy_id, current_time_millis()).await.unwrap();
|
||||
let snapshot = lease.last_manifest_snapshot.as_ref().unwrap();
|
||||
assert_eq!(snapshot.visible_proxy_seqs().collect::<Vec<_>>(), vec![3, 4, 5]);
|
||||
assert_eq!(snapshot.playlist_duration_ms, 18_000);
|
||||
lease.state = HlsAccessLeaseState::Activated;
|
||||
lease.active_until_ms = Some(current_time_millis() + 60_000);
|
||||
manager.prepare_access_lease(lease).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[expect(clippy::too_many_lines, reason = "One ordered reload, re-entry and stale-publication transaction.")]
|
||||
async fn live_window_config_changes_reentry_and_generation_races_preserve_lease_evidence() {
|
||||
use tuliprox_hls::api::{
|
||||
derive_hls_lease_manifest_snapshot, HlsLeaseManifestSnapshotInput, HlsLiveWindowView, HlsManifestRenderer,
|
||||
};
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
dto.strip.value = 1;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session =
|
||||
manager.get_or_create_session(HlsSessionKey::new(1, "races"), b"secret", current_time_millis()).await;
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&origin(0, 6, &[4])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
prepare_revisions(&mut state, &[2, 3, 4, 5]);
|
||||
state.render_and_store_manifest(current_time_millis()).unwrap();
|
||||
}
|
||||
let id = HlsAccessLeaseId("entry".into());
|
||||
let request_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
let request = || {
|
||||
try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&request_strip,
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::ZERO),
|
||||
)
|
||||
};
|
||||
let first = request().await.unwrap();
|
||||
assert_eq!(first.status(), StatusCode::OK);
|
||||
let first_body = response_body(first).await;
|
||||
let proxy = session.read().await.proxy_session_id.clone();
|
||||
let original = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap();
|
||||
let mut leases = manager.access_leases().write().await;
|
||||
if mode != HlsStartupMode::FirstReady {
|
||||
assert!(leases.try_claim_progressive_startup(&id, 2, current_time_millis()));
|
||||
}
|
||||
drop(leases);
|
||||
// A larger strip must retain the published body, never rewind to 0,1,2.
|
||||
dto.strip.value = 3;
|
||||
let mut config = app.app_config.config.load().as_ref().clone();
|
||||
config.reverse_proxy =
|
||||
Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto {
|
||||
hls_cache: Some(dto.clone()),
|
||||
..Default::default()
|
||||
}));
|
||||
app.app_config.config.store(Arc::new(config));
|
||||
manager.update_config(&app.app_config).await;
|
||||
let retained = request().await.unwrap();
|
||||
assert_eq!(retained.status(), StatusCode::OK);
|
||||
assert_eq!(response_body(retained).await, first_body);
|
||||
dto.strip.value = 0;
|
||||
let mut config = app.app_config.config.load().as_ref().clone();
|
||||
config.reverse_proxy.as_mut().unwrap().hls_cache = Some(HlsCacheConfig::from(&dto));
|
||||
app.app_config.config.store(Arc::new(config));
|
||||
manager.update_config(&app.app_config).await;
|
||||
let next = request().await.unwrap();
|
||||
assert_eq!(next.status(), StatusCode::OK);
|
||||
let _ = response_body(next).await;
|
||||
let mut lease = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap();
|
||||
assert_eq!(
|
||||
lease.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::<Vec<_>>(),
|
||||
vec![3, 4, 5]
|
||||
);
|
||||
lease.state = HlsAccessLeaseState::Idle;
|
||||
manager.prepare_access_lease(lease).await;
|
||||
let idle = request().await.unwrap();
|
||||
assert_eq!(idle.status(), StatusCode::OK);
|
||||
let _ = response_body(idle).await;
|
||||
if mode != HlsStartupMode::FirstReady {
|
||||
assert!(!manager.access_leases().write().await.try_claim_progressive_startup(
|
||||
&id,
|
||||
3,
|
||||
current_time_millis()
|
||||
));
|
||||
}
|
||||
let guard =
|
||||
manager.prepare_access_lease_manifest_publication(&id, &proxy, current_time_millis()).await.unwrap();
|
||||
let stale = {
|
||||
let state = session.read().await;
|
||||
let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap();
|
||||
let rendered = HlsManifestRenderer::render_live_window(&state, &plan, current_time_millis()).unwrap();
|
||||
derive_hls_lease_manifest_snapshot(
|
||||
&HlsLeaseManifestSnapshotInput::FastStartWindow {
|
||||
session: &state,
|
||||
plan: &plan,
|
||||
rendered: &rendered,
|
||||
materialized_body: &rendered.body,
|
||||
},
|
||||
current_time_millis(),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
};
|
||||
session.write().await.activity.origin_work_generation += 1;
|
||||
assert!(!manager
|
||||
.commit_access_lease_manifest_publication(&id, &proxy, guard, stale, current_time_millis())
|
||||
.await
|
||||
.is_committed());
|
||||
let after = manager.access_lease_response_snapshot(&id, &proxy, current_time_millis()).await.unwrap();
|
||||
assert_eq!(
|
||||
after.last_manifest_snapshot.as_ref().unwrap().visible_proxy_seqs().collect::<Vec<_>>(),
|
||||
vec![3, 4, 5]
|
||||
);
|
||||
assert_eq!(after.issued_at_ms, original.issued_at_ms);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
use super::{normal_manifest, response_body, test_app_state_with_hls_proxy, try_test_hls_cached_manifest_response};
|
||||
use crate::api::model::{
|
||||
HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey,
|
||||
SegmentCacheStatus,
|
||||
};
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use shared::model::{HlsCacheConfigDto, HlsStartupMode};
|
||||
use std::{fmt::Write as _, sync::Arc, time::Duration};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tuliprox_core::{
|
||||
model::{HlsCacheConfig, StripConfig},
|
||||
utils::current_time_millis,
|
||||
};
|
||||
use tuliprox_hls::api::SegmentFetchContext;
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::too_many_lines, reason = "One deterministic held-body scheduling and HTTP publication scenario.")]
|
||||
async fn retained_playlist_request_is_released_by_successor_headers_while_bodies_stay_open() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("origin");
|
||||
let origin = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (headers, header_gate) = tokio::sync::watch::channel(false);
|
||||
let (bodies, body_gate) = tokio::sync::watch::channel(false);
|
||||
let origin_task = tokio::spawn(async move {
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
socket = listener.accept() => {
|
||||
let (socket, _) = socket.unwrap();
|
||||
let mut headers = header_gate.clone();
|
||||
let mut bodies = body_gate.clone();
|
||||
children.spawn(async move {
|
||||
let mut socket = BufReader::new(socket);
|
||||
let mut request = String::new();
|
||||
socket.read_line(&mut request).await.unwrap();
|
||||
let head = request.contains("/103.ts ");
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
socket.read_line(&mut line).await.unwrap();
|
||||
if line == "\r\n" || line.is_empty() { break; }
|
||||
}
|
||||
let mut socket = socket.into_inner();
|
||||
if !head {
|
||||
while !*headers.borrow_and_update() { headers.changed().await.unwrap(); }
|
||||
}
|
||||
socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\n").await.unwrap();
|
||||
if head { socket.write_all(b"segment-").await.unwrap(); }
|
||||
while !*bodies.borrow_and_update() { bodies.changed().await.unwrap(); }
|
||||
socket.write_all(if head { b"body".as_slice() } else { b"segment-body".as_slice() }).await.unwrap();
|
||||
});
|
||||
}
|
||||
_ = children.join_next(), if !children.is_empty() => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
max_concurrent_segment_fetches_per_session: 3,
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = HlsStartupMode::Optimistic;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session =
|
||||
manager.get_or_create_session(HlsSessionKey::new(1, "retained-render"), b"secret", current_time_millis()).await;
|
||||
let manifest = |first: u64| {
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n");
|
||||
for seq in first..first + 3 {
|
||||
writeln!(body, "#EXTINF:4,\n{origin}/{seq}.ts").unwrap();
|
||||
}
|
||||
normal_manifest(&body)
|
||||
};
|
||||
let old_rendered_at = current_time_millis().saturating_sub(1000);
|
||||
let (old_head, new_head) = {
|
||||
let mut session = session.write().await;
|
||||
let startup = session.startup.take();
|
||||
session.apply_origin_manifest(&manifest(100)).unwrap();
|
||||
for entry in session.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: old_rendered_at };
|
||||
}
|
||||
let old_head = session.render_and_store_manifest(old_rendered_at).unwrap().first_proxy_seq;
|
||||
session.apply_origin_manifest(&manifest(103)).unwrap();
|
||||
session.startup = startup;
|
||||
let new_head = session.publishable_origin_head_proxy_seq.unwrap();
|
||||
while session.segment_prefetch_queue.pop_next().is_some() {}
|
||||
let candidates = session
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, entry)| !matches!(entry.status, SegmentCacheStatus::Ready { .. }))
|
||||
.map(|(seq, _)| *seq)
|
||||
.collect::<Vec<_>>();
|
||||
for seq in candidates {
|
||||
session.segments.get_mut(&seq).unwrap().status = SegmentCacheStatus::Discovered;
|
||||
assert!(session.queue_segment_fetch_candidate(
|
||||
seq,
|
||||
tuliprox_hls::api::SegmentFetchPriority::Demand,
|
||||
current_time_millis()
|
||||
));
|
||||
}
|
||||
(old_head, new_head)
|
||||
};
|
||||
let session_id = session.read().await.proxy_session_id.clone();
|
||||
let lease_id = HlsAccessLeaseId("waiting-reused-session".into());
|
||||
manager
|
||||
.prepare_access_lease(HlsAccessLease::pending(
|
||||
lease_id.clone(),
|
||||
HlsPlaybackFamilyKey::new("test-user", "render-gate"),
|
||||
session_id,
|
||||
"test-user".into(),
|
||||
"render-session".into(),
|
||||
1,
|
||||
"retained-render".into(),
|
||||
1,
|
||||
current_time_millis(),
|
||||
60_000,
|
||||
))
|
||||
.await;
|
||||
let client = reqwest::Client::new();
|
||||
manager
|
||||
.segment_worker_pool()
|
||||
.wake_scheduler(
|
||||
SegmentFetchContext {
|
||||
session: session.clone(),
|
||||
segment_cache: manager.segment_cache().clone(),
|
||||
segment_repair: manager.segment_repair().clone(),
|
||||
repair_access_lease_id: None,
|
||||
headers: HeaderMap::new(),
|
||||
origin_provider_session_headers: HeaderMap::new(),
|
||||
client: client.clone(),
|
||||
no_redirect_client: client,
|
||||
use_manual_redirects: false,
|
||||
origin_io: None,
|
||||
},
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if session
|
||||
.read()
|
||||
.await
|
||||
.startup
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.revisions
|
||||
.values()
|
||||
.any(|revision| revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire) > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("head prefix");
|
||||
assert_eq!(session.read().await.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, old_head);
|
||||
let request = {
|
||||
let app = app.clone();
|
||||
let session = session.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
tokio::spawn(async move {
|
||||
try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&lease_id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: shared::model::HlsStripMode::Segments, value: 0 },
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::from_secs(2))
|
||||
.requiring_newer_manifest(old_rendered_at),
|
||||
)
|
||||
.await
|
||||
})
|
||||
};
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
assert!(!request.is_finished());
|
||||
headers.send_replace(true);
|
||||
let response = tokio::time::timeout(Duration::from_secs(1), request).await.unwrap().unwrap().expect("playlist");
|
||||
let answered_at = current_time_millis();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(body.contains(&format!("{new_head:06}.ts")));
|
||||
let state = session.read().await;
|
||||
let rendered = state.last_rendered_manifest.as_ref().unwrap();
|
||||
assert!(answered_at.saturating_sub(rendered.rendered_at_ms) <= 50, "release exceeded two poll intervals");
|
||||
assert!(state.startup.as_ref().unwrap().revisions.values().all(|revision| !revision.revision().is_complete()));
|
||||
assert!(!*bodies.borrow());
|
||||
drop(state);
|
||||
bodies.send_replace(true);
|
||||
origin_task.abort();
|
||||
}
|
||||
@@ -239,6 +239,7 @@ async fn hls_provider_exhausted_without_provisioning_returns_custom_manifest() {
|
||||
|
||||
let response = super::super::hls_shared_provisioning_or_provider_exhausted_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
"hls-user",
|
||||
&input,
|
||||
@@ -282,6 +283,7 @@ async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segmen
|
||||
|
||||
let response = super::super::hls_shared_provisioning_timeline_manifest_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
&access_lease_id,
|
||||
HlsAccessLeaseState::Activated,
|
||||
@@ -315,6 +317,7 @@ async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segmen
|
||||
|
||||
let response = super::super::hls_shared_provisioning_timeline_manifest_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
&access_lease_id,
|
||||
HlsAccessLeaseState::Activated,
|
||||
@@ -342,6 +345,7 @@ async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() {
|
||||
|
||||
super::super::hls_shared_provisioning_timeline_manifest_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
&initial_lease_id,
|
||||
HlsAccessLeaseState::Activated,
|
||||
@@ -404,6 +408,7 @@ async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segment
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
super::super::hls_shared_provisioning_timeline_manifest_response(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
&access_lease_id,
|
||||
HlsAccessLeaseState::Activated,
|
||||
|
||||
@@ -548,6 +548,7 @@ async fn hls_provider_exhausted_grace_hold_waits_for_grace_period_before_retry()
|
||||
let strip = app_state.hls.proxy.strip();
|
||||
let resolution = super::super::hls_provider_connections_exhausted_manifest_resolution(
|
||||
&app_state,
|
||||
None,
|
||||
&session,
|
||||
"hls-user",
|
||||
&input,
|
||||
@@ -599,6 +600,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() {
|
||||
|
||||
let grace = super::super::hls_provider_connections_exhausted_manifest_resolution(
|
||||
&fixture.app_state,
|
||||
None,
|
||||
&session,
|
||||
"hls-user",
|
||||
&input,
|
||||
@@ -614,6 +616,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() {
|
||||
|
||||
let response = super::super::hls_provider_connections_exhausted_manifest_resolution(
|
||||
&fixture.app_state,
|
||||
None,
|
||||
&session,
|
||||
"hls-user",
|
||||
&input,
|
||||
@@ -635,6 +638,7 @@ async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() {
|
||||
assert_eq!(plan.segment_duration_ms, 10_027);
|
||||
let replay = super::super::hls_provider_connections_exhausted_manifest_resolution(
|
||||
&fixture.app_state,
|
||||
None,
|
||||
&session,
|
||||
"hls-user",
|
||||
&input,
|
||||
|
||||
@@ -99,7 +99,9 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) {
|
||||
let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await;
|
||||
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
|
||||
let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
|
||||
ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await;
|
||||
let baseline_sequences: &[u64] =
|
||||
if mode == HlsStartupMode::Conservative { &[0, 1, 2, 3, 4, 5] } else { &[3, 4, 5] };
|
||||
ready(&app, &session, baseline_sequences).await;
|
||||
let response = try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
@@ -113,7 +115,7 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) {
|
||||
.expect("complete baseline is published to the playback lease");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let first = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
deliver(&app, &session, &first, &[1000]).await;
|
||||
deliver(&app, &session, &first, &[1000 + baseline_sequences[0]]).await;
|
||||
tokio::time::timeout(WAIT, async {
|
||||
let proxy_id = session.read().await.proxy_session_id.clone();
|
||||
loop {
|
||||
@@ -123,7 +125,7 @@ async fn repaired_reduced_retry(mode: HlsStartupMode) {
|
||||
.access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis())
|
||||
.await
|
||||
.unwrap();
|
||||
if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) {
|
||||
if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(baseline_sequences[0]) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
@@ -104,7 +104,12 @@ async fn deliver_origin_segment(app: &Arc<AppState>, session: &HlsSessionHandle,
|
||||
let origin_url = {
|
||||
let s = session.read().await;
|
||||
let segment = &s.segments[&seq];
|
||||
assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window");
|
||||
assert!(
|
||||
(head..head + 6).contains(&segment.origin_key.host_local_sequence),
|
||||
"current origin window: expected {head}..{}; got {} at proxy {seq}; manifest={manifest}",
|
||||
head + 6,
|
||||
segment.origin_key.host_local_sequence
|
||||
);
|
||||
segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()
|
||||
};
|
||||
let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string();
|
||||
|
||||
@@ -62,6 +62,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn try_test_hls_cached_manif
|
||||
}
|
||||
super::super::super::try_hls_cached_manifest_response(
|
||||
app_state,
|
||||
None,
|
||||
session,
|
||||
access_lease_id,
|
||||
access_lease_state,
|
||||
|
||||
@@ -68,6 +68,7 @@ impl CanonicalOwnerHandoffFixture {
|
||||
) -> super::super::super::HlsCanonicalOwnerHandoffContext<'_> {
|
||||
let (lease_id, issued_at_ms) = &self.leases[lease_index];
|
||||
super::super::super::HlsCanonicalOwnerHandoffContext {
|
||||
fingerprint: None,
|
||||
app_state: &self.app_state,
|
||||
proxy_session_id: &self.proxy_session_id,
|
||||
access_lease_id: lease_id,
|
||||
|
||||
@@ -78,6 +78,7 @@ async fn completed_owner_resolves_new_lease_standalone_without_reusing_old_termi
|
||||
let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 };
|
||||
let mut response = Box::pin(super::super::join_hls_canonical_manifest_owner(
|
||||
super::super::HlsCanonicalOwnerHandoffContext {
|
||||
fingerprint: None,
|
||||
app_state: &app_state,
|
||||
proxy_session_id: &proxy_session_id,
|
||||
access_lease_id: &new_lease_id,
|
||||
|
||||
@@ -92,12 +92,6 @@ app_state_views! {
|
||||
app_config, recordings, events <- event_manager, http_clients, recording_capacity,
|
||||
}
|
||||
|
||||
/// The handles the HLS proxy needs: itself, plus provider allocation and
|
||||
/// session accounting.
|
||||
hls_ctx => crate::api::model::hls_cache::HlsCtx {
|
||||
app_config, hls_proxy <- hls.proxy, active_provider, connection_manager, active_users,
|
||||
}
|
||||
|
||||
/// The handles admission reads: it decides over connections and users.
|
||||
admission_ctx => tuliprox_session::admission::AdmissionCtx {
|
||||
app_config, active_users, connection_manager,
|
||||
@@ -116,3 +110,23 @@ app_state_views! {
|
||||
maintenance as std::sync::Arc<dyn tuliprox_core::model::MaintenanceGate>,
|
||||
}
|
||||
}
|
||||
|
||||
impl AppStateView for crate::api::model::hls_cache::HlsCtx {
|
||||
fn from_app_state(state: &AppState) -> Self {
|
||||
let ctx = Self {
|
||||
app_config: state.app_config.clone(),
|
||||
hls_proxy: state.hls.proxy.clone(),
|
||||
active_provider: state.active_provider.clone(),
|
||||
connection_manager: state.connection_manager.clone(),
|
||||
active_users: state.active_users.clone(),
|
||||
};
|
||||
ctx.attach_lease_user_accounting();
|
||||
ctx
|
||||
}
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
/// The HLS runtime and its authoritative session-accounting owner.
|
||||
#[must_use]
|
||||
pub fn hls_ctx(&self) -> crate::api::model::hls_cache::HlsCtx { self.view() }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Real Tuliprox process + a streaming origin. No application HTTP client is stubbed.
|
||||
use std::{
|
||||
fmt::Write as _,
|
||||
path::PathBuf,
|
||||
process::Stdio,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
|
||||
net::TcpListener,
|
||||
sync::watch,
|
||||
};
|
||||
|
||||
const BODY: &[u8] =
|
||||
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"));
|
||||
|
||||
// A configured Postman member is shared between the process scenarios.
|
||||
static SCENARIO: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
struct Fixture {
|
||||
directory: tempfile::TempDir,
|
||||
base: String,
|
||||
process: tokio::process::Child,
|
||||
origin: tokio::task::JoinHandle<()>,
|
||||
release: watch::Sender<bool>,
|
||||
release_retry: watch::Sender<bool>,
|
||||
attempts: Arc<AtomicUsize>,
|
||||
_scenario: tokio::sync::MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.process.start_kill();
|
||||
self.origin.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
#[allow(clippy::too_many_lines, reason = "Keeps the isolated process and streaming origin fixture together.")]
|
||||
async fn start() -> Self {
|
||||
let scenario = SCENARIO.lock().await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let origin_base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let manifest_base =
|
||||
std::env::var("TULIPROX_HLS_RECOVERY_MANIFEST_BASE").unwrap_or_else(|_| origin_base.clone());
|
||||
if manifest_base != origin_base {
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{manifest_base}/configure"))
|
||||
.json(&serde_json::json!({"segmentOrigin": origin_base}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 204);
|
||||
}
|
||||
let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"recovery\" group-title=\"Test\",Recovery Channel\n{manifest_base}/index.m3u8\n");
|
||||
let mut manifest = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n".to_string();
|
||||
for seq in 100..106 {
|
||||
writeln!(manifest, "#EXTINF:4,\n{origin_base}/{seq}.ts").unwrap();
|
||||
}
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let (release, released) = watch::channel(false);
|
||||
let (release_retry, retry_released) = watch::channel(false);
|
||||
let counted = attempts.clone();
|
||||
let origin = tokio::spawn(async move {
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
accepted = listener.accept() => {
|
||||
let (socket, _) = accepted.unwrap();
|
||||
let catalog = catalog.clone(); let manifest = manifest.clone();
|
||||
let attempts = counted.clone(); let mut released = released.clone();
|
||||
let mut retry_released = retry_released.clone();
|
||||
children.spawn(async move {
|
||||
let mut reader = BufReader::new(socket);
|
||||
let mut request = String::new();
|
||||
reader.read_line(&mut request).await.unwrap();
|
||||
let path = request.split_whitespace().nth(1).unwrap().to_owned();
|
||||
loop {
|
||||
let mut line = String::new(); reader.read_line(&mut line).await.unwrap();
|
||||
if line == "\r\n" || line.is_empty() { break; }
|
||||
}
|
||||
let mut socket = reader.into_inner();
|
||||
if path == "/catalog.m3u" || path == "/index.m3u8" {
|
||||
let body = if path == "/index.m3u8" { manifest } else { catalog };
|
||||
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/vnd.apple.mpegurl\r\nConnection: close\r\n\r\n", body.len());
|
||||
socket.write_all(headers.as_bytes()).await.unwrap();
|
||||
socket.write_all(body.as_bytes()).await.unwrap();
|
||||
return;
|
||||
}
|
||||
// The accelerated entry is 103,104,105. Inject the
|
||||
// failure into its first visible successor.
|
||||
let failed_attempt = path == "/104.ts" && attempts.fetch_add(1, Ordering::AcqRel) == 0;
|
||||
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: video/mp2t\r\nConnection: close\r\n\r\n", BODY.len());
|
||||
socket.write_all(headers.as_bytes()).await.unwrap();
|
||||
if failed_attempt {
|
||||
socket.write_all(b"stale-prefix").await.unwrap();
|
||||
// Hold an incomplete 200 open beyond the configured body idle timeout.
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
return;
|
||||
}
|
||||
if path == "/104.ts" {
|
||||
while !*retry_released.borrow_and_update() {
|
||||
if retry_released.changed().await.is_err() { return; }
|
||||
}
|
||||
}
|
||||
socket.write_all(&BODY[..188]).await.unwrap();
|
||||
while !*released.borrow_and_update() {
|
||||
if released.changed().await.is_err() { return; }
|
||||
}
|
||||
let _ = socket.write_all(&BODY[188..]).await;
|
||||
});
|
||||
}
|
||||
_ = children.join_next(), if !children.is_empty() => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = reserved.local_addr().unwrap().port();
|
||||
drop(reserved);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let root = directory.path();
|
||||
for name in ["web", "storage", "backup"] {
|
||||
std::fs::create_dir(root.join(name)).unwrap();
|
||||
}
|
||||
let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n origin_segment_timeout_ms: 300\n max_concurrent_segment_fetches_per_session: 3\n initial_manifest_wait_timeout_secs: 3\n startup:\n mode: optimistic\n segment_repair:\n max_level: off\n", root.join("web").display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display());
|
||||
let sources = format!("inputs:\n - name: recovery-origin\n type: m3u\n url: {origin_base}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [recovery-origin]\n targets:\n - name: recovery\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n");
|
||||
let users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Recovery test\nuser:\n - target: recovery\n credentials:\n - username: viewer\n password: test-password\n proxy: reverse\n server: local\n status: Active\n max_connections: 1\n");
|
||||
for (name, body) in [("config.yml", config), ("source.yml", sources), ("api-proxy.yml", users)] {
|
||||
std::fs::write(root.join(name), body).unwrap();
|
||||
}
|
||||
let log = std::fs::File::create(root.join("sut.log")).unwrap();
|
||||
let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox"))
|
||||
.args(["-s", "-c", "config.yml", "-i", "source.yml", "-a", "api-proxy.yml"])
|
||||
.current_dir(root)
|
||||
.stdout(Stdio::from(log.try_clone().unwrap()))
|
||||
.stderr(Stdio::from(log))
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.unwrap();
|
||||
Self {
|
||||
directory,
|
||||
base: format!("http://127.0.0.1:{port}"),
|
||||
process,
|
||||
origin,
|
||||
release,
|
||||
release_retry,
|
||||
attempts,
|
||||
_scenario: scenario,
|
||||
}
|
||||
}
|
||||
|
||||
fn log(&self) -> String { std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() }
|
||||
|
||||
async fn playback(&mut self, client: &reqwest::Client) -> String {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
assert!(self.process.try_wait().unwrap().is_none(), "SUT exited: {}", self.log());
|
||||
if let Ok(response) =
|
||||
client.get(format!("{}/m3u?username=viewer&password=test-password", self.base)).send().await
|
||||
{
|
||||
if let Ok(body) = response.text().await {
|
||||
if let Some(url) = body.lines().find(|line| line.starts_with("http")) {
|
||||
return url.to_owned();
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(tokio::time::Instant::now() < deadline, "catalog unavailable: {}", self.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_media(base: &str, body: &str, index: usize) -> String {
|
||||
let uri = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).nth(index).expect("media URI");
|
||||
reqwest::Url::parse(base).unwrap().join(uri).unwrap().to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incomplete_origin_200_recovers_same_segment_uri_without_a_playlist_refresh() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap();
|
||||
assert!(master.status().is_success(), "entry: {}", fixture.log());
|
||||
let master_body = master.text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master_body, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(playlist.status(), 200, "startup: {}", fixture.log());
|
||||
let playlist = playlist.text().await.unwrap();
|
||||
assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:3\n"), "entry follows the current live edge");
|
||||
let segment = resolve_media(&variant, &playlist, 1);
|
||||
assert_eq!(fixture.attempts.load(Ordering::Acquire), 1);
|
||||
// Exercise unexposed successor recovery after consuming this lease's one
|
||||
// progressive request. A successor selected as the first GET may now pipe
|
||||
// its prefix and therefore cannot silently switch bytes after a body drop.
|
||||
let head = client.get(resolve_media(&variant, &playlist, 0)).send().await.unwrap();
|
||||
assert_eq!(head.status(), 200, "head must stream before EOF: {}", fixture.log());
|
||||
fixture.release.send_replace(true);
|
||||
assert_eq!(head.bytes().await.unwrap().as_ref(), BODY);
|
||||
let waiting = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(waiting.status(), 503, "retry must be temporary: {}", fixture.log());
|
||||
assert!(waiting.headers().contains_key("retry-after"));
|
||||
// Make the pending response independent of the origin retry's timing.
|
||||
fixture.release_retry.send_replace(true);
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let response = client.get(&segment).send().await.unwrap();
|
||||
if response.status() == 200 {
|
||||
assert_eq!(response.bytes().await.unwrap().as_ref(), BODY);
|
||||
break;
|
||||
}
|
||||
assert_eq!(response.status(), 503, "same URI must not become 404 during recovery: {}", fixture.log());
|
||||
assert!(tokio::time::Instant::now() < deadline, "retry never recovered: {}", fixture.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
assert!(fixture.attempts.load(Ordering::Acquire) >= 2);
|
||||
let refreshed = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(refreshed.status(), 200, "recovered binding must not conflict: {}", fixture.log());
|
||||
assert!(refreshed.text().await.unwrap().contains(reqwest::Url::parse(&segment).unwrap().path()));
|
||||
for _ in 0..3 {
|
||||
let replay = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY);
|
||||
}
|
||||
assert_eq!(fixture.attempts.load(Ordering::Acquire), 2, "repaired GETs reuse the committed revision");
|
||||
assert!(fixture.log().contains("outcome=replacement_bound"));
|
||||
// Retain run evidence only when requested; test configuration and credentials are synthetic.
|
||||
if let Ok(destination) = std::env::var("TULIPROX_HLS_RECOVERY_LOG") {
|
||||
std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap();
|
||||
}
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_requested_successor_streams_before_origin_eof() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap().text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(playlist.status(), 200, "startup: {}", fixture.log());
|
||||
let playlist = playlist.text().await.unwrap();
|
||||
// Skip both the playlist head and the deliberately failing first successor.
|
||||
let segment = resolve_media(&variant, &playlist, 2);
|
||||
let mut response = tokio::time::timeout(Duration::from_secs(2), client.get(&segment).send())
|
||||
.await
|
||||
.expect("first successor GET must not wait for origin EOF")
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200, "successor: {}", fixture.log());
|
||||
assert!(!response.headers().contains_key("content-length"));
|
||||
let prefix = response.chunk().await.unwrap().expect("progressive prefix");
|
||||
assert_eq!(prefix.as_ref(), &BODY[..188]);
|
||||
assert!(!*fixture.release.borrow(), "origin EOF is still held");
|
||||
fixture.release.send_replace(true);
|
||||
fixture.release_retry.send_replace(true);
|
||||
let mut bytes = prefix.to_vec();
|
||||
while let Some(chunk) = response.chunk().await.unwrap() {
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
assert_eq!(bytes, BODY);
|
||||
let replay = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY);
|
||||
if let Ok(destination) = std::env::var("TULIPROX_HLS_FIRST_REQUEST_LOG") {
|
||||
std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap();
|
||||
}
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exposed_failure_allows_a_new_player_with_user_connection_limit_one() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
|
||||
let segment = resolve_media(&variant, &playlist, 1);
|
||||
// The first full GET deliberately selects the successor, exercising the
|
||||
// integrated piping policy and exposing this attempt's prefix.
|
||||
let exposed = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(exposed.status(), 200, "first successor GET must pipe: {}", fixture.log());
|
||||
fixture.release.send_replace(true);
|
||||
assert!(exposed.bytes().await.is_err(), "exposed truncated body must not become successful EOF");
|
||||
let ended = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(ended.status(), 404, "old lease must end: {}", fixture.log());
|
||||
fixture.release_retry.send_replace(true);
|
||||
let new_player =
|
||||
reqwest::Client::builder().user_agent("new-reentry-player").timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let master = new_player.get(&entry).send().await.unwrap();
|
||||
assert_eq!(master.status(), 200, "counted user reservation blocked reentry: {}", fixture.log());
|
||||
let body = master.text().await.unwrap();
|
||||
assert!(body.contains("#EXT-X-STREAM-INF"), "new entry must reach live playback: {body}");
|
||||
let new_variant = resolve_media(&entry, &body, 0);
|
||||
assert_ne!(new_variant, variant);
|
||||
let previous_session = reqwest::Url::parse(&variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned();
|
||||
let current_session =
|
||||
reqwest::Url::parse(&new_variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned();
|
||||
assert_eq!(previous_session, current_session, "the shared origin session stays alive");
|
||||
// Reentry releases the user slot before the origin retry finishes. The
|
||||
// pending lease may still wait for that retry's publishable media.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
let playlist = loop {
|
||||
let response = new_player.get(&new_variant).send().await.unwrap();
|
||||
if response.status() == 200 {
|
||||
break response;
|
||||
}
|
||||
// Before its first publication, a lease can use the unavailable-channel
|
||||
// fallback (404 without a configured custom stream) or a temporary 503.
|
||||
assert!(
|
||||
matches!(response.status().as_u16(), 404 | 503),
|
||||
"new live playlist rejected with {}: {}",
|
||||
response.status(),
|
||||
fixture.log()
|
||||
);
|
||||
assert!(tokio::time::Instant::now() < deadline, "new live playlist never recovered: {}", fixture.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
};
|
||||
let new_segment = resolve_media(&new_variant, &playlist.text().await.unwrap(), 1);
|
||||
let repaired = new_player.get(new_segment).send().await.unwrap();
|
||||
assert_eq!(repaired.status(), 200, "new player cannot use the successful revision: {}", fixture.log());
|
||||
assert_eq!(repaired.bytes().await.unwrap().as_ref(), BODY);
|
||||
assert_eq!(client.get(&segment).send().await.unwrap().status(), 404);
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
@@ -4,8 +4,9 @@ use super::{
|
||||
selection::{
|
||||
capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment,
|
||||
fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate,
|
||||
remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries,
|
||||
session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids,
|
||||
remove_map_entry, remove_obsolete_deferred_heads, remove_obsolete_segment_metadata, remove_segment_entry,
|
||||
remove_stale_queue_entries, session_cache_size, take_expired_transient_objects,
|
||||
unprotected_unreferenced_map_ids,
|
||||
},
|
||||
CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome,
|
||||
HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle,
|
||||
@@ -203,6 +204,7 @@ impl HlsGarbageCollector {
|
||||
report.stale_queue_entries_removed =
|
||||
report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session));
|
||||
|
||||
remove_obsolete_segment_metadata(session, &protected_set_for_cache_reclamation(session, cache, None, None));
|
||||
remove_obsolete_deferred_heads(session, cache, None, None, deletions);
|
||||
let transient_before = session.transient.resources.len();
|
||||
let protected = protected_set_for_cache_reclamation(session, cache, None, None);
|
||||
|
||||
@@ -75,6 +75,21 @@ impl HlsGarbageCollector {
|
||||
continue;
|
||||
};
|
||||
let id = session.proxy_session_id.clone();
|
||||
if let Some(leases) = &mut lease_guard {
|
||||
session.live_window_demands.retain(|lease_id, demand| {
|
||||
leases.response_snapshot(lease_id, &id, tuliprox_core::utils::current_time_millis()).is_some_and(
|
||||
|lease| {
|
||||
lease.issued_at_ms == demand.issued_at_ms
|
||||
&& matches!(
|
||||
lease.state,
|
||||
crate::HlsAccessLeaseState::Pending
|
||||
| crate::HlsAccessLeaseState::Activated
|
||||
| crate::HlsAccessLeaseState::Idle
|
||||
)
|
||||
},
|
||||
)
|
||||
});
|
||||
}
|
||||
let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq);
|
||||
let crate::HlsSession { startup, segments, .. } = &mut *session;
|
||||
if let Some(startup) = startup {
|
||||
|
||||
@@ -147,13 +147,36 @@ pub(super) fn remove_stale_queue_entries(session: &mut HlsSession) -> usize {
|
||||
removed
|
||||
}
|
||||
|
||||
/// Fast-start origin mappings are needed only while their origin window or an
|
||||
/// existing publication, reader or cache mutation still owns them.
|
||||
pub(super) fn remove_obsolete_segment_metadata(session: &mut HlsSession, protected: &ProtectedSet) {
|
||||
if !session.uses_fast_start_live_window() {
|
||||
return;
|
||||
}
|
||||
let Some(origin) = session.origin_window_proxy_seqs.as_ref() else { return };
|
||||
let origin = origin.iter().copied().collect::<HashSet<_>>();
|
||||
let obsolete = session
|
||||
.segments
|
||||
.iter()
|
||||
.filter_map(|(seq, entry)| {
|
||||
(matches!(entry.status, SegmentCacheStatus::Discovered)
|
||||
&& !origin.contains(seq)
|
||||
&& !protected.segment_proxy_seqs.contains(seq))
|
||||
.then_some(*seq)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for seq in obsolete {
|
||||
let _ = remove_segment_entry(session, seq);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn duration_expired_head_segment(
|
||||
session: &HlsSession,
|
||||
protected: &ProtectedSet,
|
||||
policy: &GarbageCollectionPolicy,
|
||||
now_ms: u64,
|
||||
) -> Option<u64> {
|
||||
let (proxy_seq, segment) = session.segments.iter().next()?;
|
||||
let (proxy_seq, segment) = fifo_head_segment(session)?;
|
||||
if protected.segment_proxy_seqs.contains(proxy_seq) {
|
||||
return None;
|
||||
}
|
||||
@@ -177,7 +200,7 @@ pub(super) fn fifo_head_size_candidate(
|
||||
session: &HlsSession,
|
||||
protected: &ProtectedSet,
|
||||
) -> Option<SegmentDeleteCandidate> {
|
||||
let (proxy_seq, segment) = session.segments.iter().next()?;
|
||||
let (proxy_seq, segment) = fifo_head_segment(session)?;
|
||||
if protected.segment_proxy_seqs.contains(proxy_seq) {
|
||||
return None;
|
||||
}
|
||||
@@ -191,6 +214,12 @@ pub(super) fn fifo_head_size_candidate(
|
||||
})
|
||||
}
|
||||
|
||||
/// Origin mappings without cache objects are not FIFO barriers. All other
|
||||
/// statuses retain the existing protection and expiry checks at the media head.
|
||||
fn fifo_head_segment(session: &HlsSession) -> Option<(&u64, &super::super::SegmentEntry)> {
|
||||
session.segments.iter().find(|(_, segment)| !matches!(segment.status, SegmentCacheStatus::Discovered))
|
||||
}
|
||||
|
||||
fn segment_last_relevant_at_ms(segment: &super::super::SegmentEntry) -> Option<u64> {
|
||||
let status_at = match segment.status {
|
||||
SegmentCacheStatus::Ready { ready_at_ms, .. } => Some(ready_at_ms),
|
||||
@@ -276,19 +305,30 @@ pub(super) fn capacity_reclamation_evidence(
|
||||
|
||||
pub(super) fn remove_segment_entry(session: &mut HlsSession, proxy_seq: u64) -> Option<SegmentCacheKey> {
|
||||
let segment = session.segments.remove(&proxy_seq)?;
|
||||
// Media outside the selected window can be evicted, but the current
|
||||
// origin's mapping must survive until its next slide. Otherwise an older,
|
||||
// unselected URI is appended anew ahead of already published overlaps.
|
||||
let retain_origin_metadata =
|
||||
session.uses_fast_start_live_window() && session.live_origin_sequences().contains(&proxy_seq);
|
||||
if let Some(startup) = &mut session.startup {
|
||||
startup.revisions.remove(&proxy_seq);
|
||||
}
|
||||
let removed_ready_media = matches!(segment.status, SegmentCacheStatus::Ready { .. });
|
||||
session.segment_prefetch_queue.remove(proxy_seq);
|
||||
session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq);
|
||||
if segment.discontinuity_before {
|
||||
if retain_origin_metadata {
|
||||
let mut metadata = segment.clone();
|
||||
metadata.status = SegmentCacheStatus::Discovered;
|
||||
session.segments.insert(proxy_seq, metadata);
|
||||
} else {
|
||||
session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq);
|
||||
}
|
||||
if segment.discontinuity_before && !retain_origin_metadata {
|
||||
session.discontinuity_sequence = session.discontinuity_sequence.saturating_add(1);
|
||||
}
|
||||
if removed_ready_media {
|
||||
session.advance_media_readiness_generation();
|
||||
}
|
||||
if session.publishable_origin_head_proxy_seq == Some(proxy_seq) {
|
||||
if session.publishable_origin_head_proxy_seq == Some(proxy_seq) && !retain_origin_metadata {
|
||||
session.publishable_origin_head_proxy_seq = session
|
||||
.segments
|
||||
.range(proxy_seq.saturating_add(1)..)
|
||||
|
||||
@@ -292,3 +292,317 @@ fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
|
||||
assert!(remove_segment_entry(&mut session, 7).is_some());
|
||||
assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500);
|
||||
}
|
||||
|
||||
fn install_fast_start(session: &mut HlsSession) {
|
||||
let config = tuliprox_core::model::HlsStartupConfig {
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
..Default::default()
|
||||
};
|
||||
session.startup = Some(crate::HlsSessionStartup {
|
||||
config: config.clone(),
|
||||
first_data_timeout_ms: 1_000,
|
||||
worker: std::sync::Weak::new(),
|
||||
access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())),
|
||||
revisions: std::collections::BTreeMap::new(),
|
||||
policy_fixed: true,
|
||||
store: Arc::new(crate::SegmentRevisionStore::default()),
|
||||
budget: crate::ProgressiveBudgetManager::new(config),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_only_prefix_does_not_block_reclaimable_ready_segments() {
|
||||
let mut session = HlsSession::new(HlsSessionKey::new(1, "review-fifo-prefix"), b"secret", 0);
|
||||
session.apply_origin_manifest(&six_segment_manifest()).unwrap();
|
||||
install_fast_start(&mut session);
|
||||
for seq in [3, 4, 5] {
|
||||
session.segments.get_mut(&seq).unwrap().status =
|
||||
SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: 0 };
|
||||
}
|
||||
session.apply_origin_manifest(&normal_manifest(
|
||||
"#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:7\n#EXTINF:4,\n7.ts\n#EXTINF:4,\n8.ts\n#EXTINF:4,\n9.ts\n#EXTINF:4,\n10.ts\n#EXTINF:4,\n11.ts\n#EXTINF:4,\n12.ts\n"
|
||||
)).unwrap();
|
||||
let protected = ProtectedSet::from_session(&session);
|
||||
assert!(!protected.segment_proxy_seqs.contains(&3));
|
||||
assert!(matches!(session.segments[&0].status, SegmentCacheStatus::Discovered));
|
||||
let reclaim = crate::gc::selection::fifo_head_size_candidate(&session, &protected);
|
||||
assert_eq!(
|
||||
reclaim.map(|entry| entry.proxy_seq),
|
||||
Some(3),
|
||||
"unfetched metadata 0..2 must not prevent reclamation of expired media 3..5"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000),
|
||||
Some(3)
|
||||
);
|
||||
// The first actual media object still blocks FIFO if it is protected or unexpired.
|
||||
assert_eq!(crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 0), None);
|
||||
let mut protected = protected;
|
||||
protected.segment_proxy_seqs.insert(3);
|
||||
assert!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).is_none());
|
||||
assert_eq!(
|
||||
crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reinserted_metadata_does_not_block_subsequent_cache_deletions() {
|
||||
let mut session = HlsSession::new(HlsSessionKey::new(1, "review-fifo-delete"), b"secret", 0);
|
||||
session.apply_origin_manifest(&six_segment_manifest()).unwrap();
|
||||
install_fast_start(&mut session);
|
||||
for entry in session.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: 0 };
|
||||
}
|
||||
let protected = ProtectedSet::from_session(&session);
|
||||
assert_eq!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).unwrap().proxy_seq, 0);
|
||||
remove_segment_entry(&mut session, 0).unwrap();
|
||||
let protected = ProtectedSet::from_session(&session);
|
||||
assert!(!protected.segment_proxy_seqs.contains(&1));
|
||||
let next = crate::gc::selection::fifo_head_size_candidate(&session, &protected);
|
||||
assert_eq!(
|
||||
next.map(|entry| entry.proxy_seq),
|
||||
Some(1),
|
||||
"preserving origin mapping must not prevent further capacity reclamation"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000),
|
||||
Some(1)
|
||||
);
|
||||
for seq in [1, 2] {
|
||||
remove_segment_entry(&mut session, seq).unwrap();
|
||||
assert!(matches!(session.segments[&seq].status, SegmentCacheStatus::Discovered));
|
||||
}
|
||||
let protected = ProtectedSet::from_session(&session);
|
||||
assert!(crate::gc::selection::fifo_head_size_candidate(&session, &protected).is_none());
|
||||
assert_eq!(
|
||||
crate::gc::selection::duration_expired_head_segment(&session, &protected, &test_policy(), 100_000),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fast_start_gc_reclaims_multiple_objects_while_retaining_origin_mappings() {
|
||||
for size_pressure in [false, true] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
update_gc_policy(&gc, |policy| {
|
||||
policy.session_idle_timeout_ms = u64::MAX;
|
||||
if size_pressure {
|
||||
policy.cache_bytes_per_session = 36;
|
||||
}
|
||||
});
|
||||
let keys = {
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&six_segment_manifest()).unwrap();
|
||||
install_fast_start(&mut state);
|
||||
for entry in state.segments.values_mut() {
|
||||
gc.cache.write_bytes_and_commit(&entry.cache_key, b"segment-body").await.unwrap();
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 };
|
||||
}
|
||||
state.segments.values().map(|entry| entry.cache_key.clone()).collect::<Vec<_>>()
|
||||
};
|
||||
let report = gc.run_once(if size_pressure { 0 } else { 100_000 }).await.unwrap();
|
||||
assert_eq!(report.segments_deleted_duration, if size_pressure { 0 } else { 3 });
|
||||
assert_eq!(report.segments_deleted_size_session, if size_pressure { 3 } else { 0 });
|
||||
let state = session.read().await;
|
||||
assert_eq!(state.origin_to_proxy.len(), 6);
|
||||
for seq in 0..3 {
|
||||
assert!(matches!(state.segments[&seq].status, SegmentCacheStatus::Discovered));
|
||||
}
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
assert_eq!(gc.cache.metadata(key).await.unwrap().is_some(), index >= 3);
|
||||
}
|
||||
drop(state);
|
||||
session.write().await.apply_origin_manifest(&rolling_metadata_manifest(7)).unwrap();
|
||||
gc.run_once(0).await.unwrap();
|
||||
let state = session.read().await;
|
||||
for seq in 0..3 {
|
||||
assert!(!state.segments.contains_key(&seq));
|
||||
assert!(!state.origin_to_proxy.values().any(|mapped| *mapped == seq));
|
||||
}
|
||||
assert!((6..12).all(|seq| state.segments.contains_key(&seq)));
|
||||
}
|
||||
}
|
||||
|
||||
fn rolling_metadata_manifest(first: u64) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest {
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n");
|
||||
for seq in first..first + 6 {
|
||||
writeln!(body, "#EXTINF:4,\n{seq}.ts").unwrap();
|
||||
}
|
||||
normal_manifest(&body)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn obsolete_metadata_stays_bounded_across_one_hundred_live_entries() {
|
||||
use shared::model::{HlsCacheConfigDto, HlsStartupMode, Secs};
|
||||
use tuliprox_core::model::HlsCacheConfig;
|
||||
|
||||
for mode in [HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
cache_duration: Secs::new(1),
|
||||
session_idle_timeout: Secs::new(1_000_000),
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = mode;
|
||||
let manager = crate::HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto));
|
||||
let session = manager.get_or_create_session(HlsSessionKey::new(1, "metadata-soak"), b"secret", 0).await;
|
||||
for round in 0..100 {
|
||||
let first = round * 6;
|
||||
let keys = {
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest_for_host(&rolling_metadata_manifest(first), 1).unwrap();
|
||||
let plan = state.plan_live_window(crate::live_window::HlsLiveWindowView::Entry).unwrap();
|
||||
assert_eq!(plan.visible, vec![first + 3, first + 4, first + 5]);
|
||||
plan.visible
|
||||
.iter()
|
||||
.map(|seq| {
|
||||
let entry = state.segments.get_mut(seq).unwrap();
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 };
|
||||
entry.cache_key.clone()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for key in keys {
|
||||
manager.segment_cache().write_bytes_and_commit(&key, b"segment-body").await.unwrap();
|
||||
}
|
||||
manager.run_garbage_collection_once(100_000 + round).await.unwrap();
|
||||
let mut state = session.write().await;
|
||||
assert_eq!(state.segments.keys().copied().collect::<Vec<_>>(), (first..first + 6).collect::<Vec<_>>());
|
||||
assert_eq!(state.origin_to_proxy.len(), 6);
|
||||
assert!(state.origin_to_proxy.values().all(|seq| (first..first + 6).contains(seq)));
|
||||
assert_eq!(
|
||||
state.segments.values().filter(|entry| matches!(entry.status, SegmentCacheStatus::Discovered)).count(),
|
||||
3
|
||||
);
|
||||
// Refreshing the same origin must reuse even its unfetched mappings.
|
||||
state.apply_origin_manifest_for_host(&rolling_metadata_manifest(first), 1).unwrap();
|
||||
assert_eq!(state.live_origin_sequences(), (first..first + 6).collect::<Vec<_>>());
|
||||
assert_eq!(state.proxy_next_seq, Some(first + 6));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn obsolete_metadata_is_removed_only_after_publication_and_reader_protection_release() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let (gc, session) = gc_with_session(&directory).await;
|
||||
let lease_id = HlsAccessLeaseId("metadata-owner".into());
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
state.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap();
|
||||
for entry in state.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 };
|
||||
}
|
||||
state.render_and_store_manifest(0).unwrap();
|
||||
install_fast_start(&mut state);
|
||||
for entry in state.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Discovered;
|
||||
}
|
||||
state.apply_origin_manifest(&rolling_metadata_manifest(6)).unwrap();
|
||||
state.segments.get_mut(&0).unwrap().access.reader_started(0);
|
||||
state.live_window_demands.insert(
|
||||
lease_id.clone(),
|
||||
crate::live_window::HlsLiveWindowDemand {
|
||||
issued_at_ms: 0,
|
||||
view: crate::live_window::HlsLiveWindowView::Entry,
|
||||
published: vec![1],
|
||||
},
|
||||
);
|
||||
state.install_terminal_tail_protection(
|
||||
lease_id.clone(),
|
||||
HlsTerminalTailProtection {
|
||||
generation: HlsTerminalTailGeneration(1),
|
||||
base_proxy_seqs: Arc::from([2]),
|
||||
key_bindings: Arc::from([]),
|
||||
},
|
||||
);
|
||||
}
|
||||
gc.run_once(0).await.unwrap();
|
||||
assert_eq!(session.read().await.segments.len(), 12);
|
||||
session.write().await.last_rendered_manifest = None;
|
||||
gc.run_once(0).await.unwrap();
|
||||
{
|
||||
let mut state = session.write().await;
|
||||
assert_eq!(state.segments.keys().copied().collect::<Vec<_>>(), vec![0, 1, 2, 6, 7, 8, 9, 10, 11]);
|
||||
assert_eq!(state.origin_to_proxy.len(), 9);
|
||||
state.segments.get_mut(&0).unwrap().access.reader_finished();
|
||||
assert!(state.live_window_demands.remove(&lease_id).is_some());
|
||||
assert!(state.remove_terminal_tail_protection(&lease_id).is_some());
|
||||
}
|
||||
gc.run_once(0).await.unwrap();
|
||||
let state = session.read().await;
|
||||
assert_eq!(state.segments.keys().copied().collect::<Vec<_>>(), (6..12).collect::<Vec<_>>());
|
||||
assert_eq!(state.origin_to_proxy.len(), 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obsolete_metadata_cleanup_preserves_non_fast_start_timeline_mappings() {
|
||||
for archive in [false, true] {
|
||||
let mut session = HlsSession::new(HlsSessionKey::new(1, "retained-metadata"), b"secret", 0);
|
||||
session.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap();
|
||||
if archive {
|
||||
install_fast_start(&mut session);
|
||||
session.origin_source.archive_reference = Some(1_784_898_000);
|
||||
}
|
||||
session.segments.get_mut(&0).unwrap().discontinuity_before = true;
|
||||
session.discontinuity_sequence = 7;
|
||||
session.apply_origin_manifest(&rolling_metadata_manifest(6)).unwrap();
|
||||
assert!(!session.uses_fast_start_live_window());
|
||||
let protected = ProtectedSet::from_session(&session);
|
||||
assert!(!protected.segment_proxy_seqs.contains(&0));
|
||||
let mappings = session.origin_to_proxy.clone();
|
||||
let sequences = session.segments.keys().copied().collect::<Vec<_>>();
|
||||
let head = session.publishable_origin_head_proxy_seq;
|
||||
let next = session.proxy_next_seq;
|
||||
|
||||
crate::gc::selection::remove_obsolete_segment_metadata(&mut session, &protected);
|
||||
|
||||
assert_eq!(session.origin_to_proxy, mappings, "archive={archive}");
|
||||
assert_eq!(session.segments.keys().copied().collect::<Vec<_>>(), sequences);
|
||||
assert_eq!(session.discontinuity_sequence, 7);
|
||||
assert_eq!(session.publishable_origin_head_proxy_seq, head);
|
||||
let mut overlapping = rolling_metadata_manifest(0);
|
||||
overlapping.segments.extend(rolling_metadata_manifest(6).segments);
|
||||
session.apply_origin_manifest(&overlapping).unwrap();
|
||||
assert_eq!(session.proxy_next_seq, next, "an overlapping URI must retain its original proxy sequence");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obsolete_metadata_cleanup_requires_origin_evidence_and_preserves_other_cache_states() {
|
||||
use crate::gc::{policy::protected_set_for_cache_reclamation, selection::remove_obsolete_segment_metadata};
|
||||
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let cache = HlsSegmentCache::with_cache_path(directory.path());
|
||||
let mut session = HlsSession::new(HlsSessionKey::new(1, "metadata-states"), b"secret", 0);
|
||||
session.apply_origin_manifest(&rolling_metadata_manifest(0)).unwrap();
|
||||
install_fast_start(&mut session);
|
||||
session.origin_window_proxy_seqs = None;
|
||||
remove_obsolete_segment_metadata(&mut session, &ProtectedSet::default());
|
||||
assert_eq!(session.segments.len(), 6);
|
||||
session.origin_window_proxy_seqs = Some(Vec::new());
|
||||
for (seq, status) in [
|
||||
SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 },
|
||||
SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::Demand, started_at_ms: 0 },
|
||||
SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Demand, queued_at_ms: 0 },
|
||||
SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 0 },
|
||||
SegmentCacheStatus::FailedRetryable { failed_at_ms: 0, retry_after_ms: 0 },
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
session.segments.get_mut(&u64::try_from(seq).unwrap()).unwrap().status = status;
|
||||
}
|
||||
let excluded = cache.object_path(&session.segments[&5].cache_key);
|
||||
let protected = protected_set_for_cache_reclamation(&session, &cache, Some(&excluded), None);
|
||||
remove_obsolete_segment_metadata(&mut session, &protected);
|
||||
assert_eq!(session.segments.len(), 6);
|
||||
let protected = protected_set_for_cache_reclamation(&session, &cache, None, None);
|
||||
remove_obsolete_segment_metadata(&mut session, &protected);
|
||||
assert!(!session.segments.contains_key(&5));
|
||||
assert_eq!(session.segments.len(), 5);
|
||||
assert_eq!(session.origin_to_proxy.len(), 5);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ pub struct HlsCtx {
|
||||
}
|
||||
|
||||
impl HlsCtx {
|
||||
/// Give request-independent lease completion its authoritative user registry.
|
||||
pub fn attach_lease_user_accounting(&self) { self.hls_proxy.bind_lease_user_accounting(&self.active_users); }
|
||||
/// The subset admission reads: it decides over connections and users, not
|
||||
/// over the cache.
|
||||
pub fn admission_ctx(&self) -> tuliprox_session::admission::AdmissionCtx {
|
||||
@@ -38,6 +40,7 @@ impl HlsCtx {
|
||||
|
||||
/// A weak handle, for state the proxy itself owns.
|
||||
pub fn downgrade(&self) -> WeakHlsCtx {
|
||||
self.attach_lease_user_accounting();
|
||||
WeakHlsCtx {
|
||||
app_config: Arc::clone(&self.app_config),
|
||||
hls_proxy: Arc::downgrade(&self.hls_proxy),
|
||||
@@ -128,12 +131,14 @@ impl HlsCtx {
|
||||
None,
|
||||
));
|
||||
|
||||
Self {
|
||||
let ctx = Self {
|
||||
hls_proxy: Arc::new(crate::manager::HlsProxyManager::new()),
|
||||
app_config: Arc::clone(&app_config),
|
||||
active_provider,
|
||||
connection_manager,
|
||||
active_users,
|
||||
}
|
||||
};
|
||||
ctx.attach_lease_user_accounting();
|
||||
ctx
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,11 +327,13 @@ mod identity;
|
||||
mod lifecycle;
|
||||
mod publication;
|
||||
mod revision_bindings;
|
||||
mod revision_recovery;
|
||||
mod runtime_policy;
|
||||
mod terminal;
|
||||
mod timing;
|
||||
|
||||
pub use self::identity::new_hls_access_lease_id;
|
||||
pub(crate) use self::revision_recovery::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome};
|
||||
use self::{
|
||||
error::HlsAvailabilityEvidenceAdvanceError,
|
||||
identity::HlsMediaLeasePlaybackIdentity,
|
||||
|
||||
@@ -26,6 +26,12 @@ pub(super) enum HlsMediaLeasePlaybackIdentity {
|
||||
|
||||
impl HlsLeaseManifestPublicationGuard {
|
||||
pub(crate) const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms }
|
||||
pub(crate) const fn media_identity(self) -> HlsMediaLeaseIdentity {
|
||||
HlsMediaLeaseIdentity {
|
||||
issued_at_ms: self.issued_at_ms,
|
||||
playback: HlsMediaLeasePlaybackIdentity::Live { admission_generation: self.admission_generation },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsAccessLease {
|
||||
|
||||
@@ -202,4 +202,6 @@ impl HlsAccessLeaseStore {
|
||||
|
||||
fn manifest_source_is_regressive(current: &HlsLeaseManifestSnapshot, candidate: &HlsLeaseManifestSnapshot) -> bool {
|
||||
candidate.source_commit_identity.commit_generation() < current.source_commit_identity.commit_generation()
|
||||
|| (candidate.startup_revisions.as_ref().is_some_and(|publication| publication.window_plan.is_some())
|
||||
&& candidate.first_proxy_seq < current.first_proxy_seq)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::{
|
||||
HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode,
|
||||
ProxySessionId,
|
||||
lease_state_allows_use, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot,
|
||||
HlsLeasePlaybackMode, ProxySessionId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -50,14 +50,19 @@ impl HlsAccessLeaseStore {
|
||||
) -> Option<super::super::SegmentRevisionGuard> {
|
||||
self.by_lease_id.get(lease_id)?.revision_bindings.as_ref()?.revisions.get(&seq).cloned()
|
||||
}
|
||||
/// Reserves the lease's single startup opportunity for its first authorized
|
||||
/// full-segment request, including a cache hit. The caller excludes partial
|
||||
/// ranges and holds this lock across lease-identity validation and lookup.
|
||||
/// Bindings include still-retained URIs from earlier published windows.
|
||||
pub fn try_claim_progressive_startup(&mut self, lease_id: &HlsAccessLeaseId, seq: u64, now_ms: u64) -> bool {
|
||||
let Some(lease) = self.by_lease_id.get_mut(lease_id) else {
|
||||
return false;
|
||||
};
|
||||
if lease.playback_mode != HlsLeasePlaybackMode::Live
|
||||
if !lease_state_allows_use(lease.state)
|
||||
|| lease.playback_mode != HlsLeasePlaybackMode::Live
|
||||
|| lease.progressive_startup_claimed
|
||||
|| lease.validity_due_at_ms() <= now_ms
|
||||
|| lease.last_manifest_snapshot.as_ref().is_none_or(|snapshot| snapshot.first_proxy_seq != seq)
|
||||
|| lease.revision_bindings.as_ref().is_none_or(|bindings| !bindings.revisions.contains_key(&seq))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -74,7 +79,9 @@ pub(super) fn commit_lease_revision_bindings(lease: &mut HlsAccessLease, snapsho
|
||||
|| current.is_some_and(|current| {
|
||||
current.retired_through.is_some_and(|retired| *seq <= retired)
|
||||
|| current.revisions.get(seq).is_some_and(|bound| {
|
||||
bound != revision && !bound.revision().is_replaceable_in(publication.mode)
|
||||
bound != revision
|
||||
&& (!bound.revision().is_replaceable_in(publication.mode)
|
||||
|| revision.revision().id() <= bound.revision().id())
|
||||
})
|
||||
})
|
||||
}) {
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! One transaction for repairing an already published segment binding.
|
||||
use super::{
|
||||
HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsAccessLeaseState, HlsAccessLeaseStore, HlsLeasePlaybackMode,
|
||||
HlsMediaLeaseIdentity,
|
||||
};
|
||||
use crate::{HlsSession, SegmentCacheStatus, SegmentRevisionGuard, SegmentRevisionState};
|
||||
use std::sync::{atomic::Ordering, Arc};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct HlsRevisionRecoveryGuard {
|
||||
identity: HlsMediaLeaseIdentity,
|
||||
snapshot_generation: u64,
|
||||
pub(crate) revision: SegmentRevisionGuard,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HlsRevisionRecoveryOutcome {
|
||||
Unchanged,
|
||||
ReplacementBound,
|
||||
RetryPending,
|
||||
ReentryRequired,
|
||||
}
|
||||
|
||||
impl HlsRevisionRecoveryGuard {
|
||||
pub(crate) fn completion_identity(&self) -> (u64, u64) {
|
||||
(self.identity.lease_issued_at_ms(), self.snapshot_generation)
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsAccessLeaseStore {
|
||||
pub(crate) fn revision_recovery_guard(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
seq: u64,
|
||||
) -> Option<HlsRevisionRecoveryGuard> {
|
||||
let lease = self.by_lease_id.get(lease_id)?;
|
||||
Some(HlsRevisionRecoveryGuard {
|
||||
identity: lease.media_identity()?,
|
||||
snapshot_generation: lease.manifest_snapshot_generation,
|
||||
revision: self.published_segment_revision(lease_id, seq)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn revision_recovery_is_current(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
expected.identity.is_live()
|
||||
&& !session.is_gc_marked_for_removal()
|
||||
&& self.media_identity_is_current(lease_id, &session.proxy_session_id, expected.identity, now_ms)
|
||||
&& self.by_lease_id.get(lease_id).is_some_and(|lease| {
|
||||
lease.manifest_snapshot_generation == expected.snapshot_generation
|
||||
&& self.published_segment_revision(lease_id, expected.revision.revision().key.proxy_seq).as_ref()
|
||||
== Some(&expected.revision)
|
||||
})
|
||||
}
|
||||
|
||||
/// Caller holds the lease store and session locks in that order. The guard
|
||||
/// fences the exact lease incarnation, snapshot and old revision together.
|
||||
pub(crate) fn recover_revision_binding(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> HlsRevisionRecoveryOutcome {
|
||||
use HlsRevisionRecoveryOutcome::{ReentryRequired, ReplacementBound, RetryPending, Unchanged};
|
||||
if !self.revision_recovery_is_current(lease_id, expected, session, now_ms)
|
||||
|| !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed)
|
||||
{
|
||||
return Unchanged;
|
||||
}
|
||||
let old = expected.revision.revision();
|
||||
let seq = old.key.proxy_seq;
|
||||
if old.was_exposed() {
|
||||
return ReentryRequired;
|
||||
}
|
||||
if !old.origin_binding_is_current(session) {
|
||||
return ReentryRequired;
|
||||
}
|
||||
let mode = self.by_lease_id.get(lease_id).and_then(|lease| {
|
||||
lease.last_manifest_snapshot.as_ref()?.startup_revisions.as_ref().map(|publication| publication.mode)
|
||||
});
|
||||
if !mode.is_some_and(|mode| old.is_replaceable_in(mode)) {
|
||||
return ReentryRequired;
|
||||
}
|
||||
let candidate = session.startup.as_ref().and_then(|startup| startup.revisions.get(&seq));
|
||||
let replacement = candidate.filter(|candidate| {
|
||||
*candidate != &expected.revision
|
||||
&& candidate.revision().key.proxy_seq == seq
|
||||
&& candidate.revision().id() > old.id()
|
||||
&& candidate.revision().key.kind == old.key.kind
|
||||
&& candidate.revision().origin_binding_is_current(session)
|
||||
&& candidate.revision().is_complete()
|
||||
&& session
|
||||
.segments
|
||||
.get(&seq)
|
||||
.is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. }))
|
||||
});
|
||||
let Some(replacement) = replacement else {
|
||||
return if session.segments.get(&seq).is_some_and(|entry| {
|
||||
matches!(
|
||||
entry.status,
|
||||
SegmentCacheStatus::Discovered
|
||||
| SegmentCacheStatus::Queued { .. }
|
||||
| SegmentCacheStatus::Fetching { .. }
|
||||
| SegmentCacheStatus::FailedRetryable { .. }
|
||||
| SegmentCacheStatus::CapacityDeferred { .. }
|
||||
)
|
||||
}) {
|
||||
RetryPending
|
||||
} else {
|
||||
ReentryRequired
|
||||
};
|
||||
};
|
||||
if self.availability_evidence_can_advance().is_err() {
|
||||
return RetryPending;
|
||||
}
|
||||
let Some(lease) = self.by_lease_id.get_mut(lease_id) else { return Unchanged };
|
||||
let Some(generation) = lease.manifest_snapshot_generation.checked_add(1) else { return RetryPending };
|
||||
let Some(bindings) = lease.revision_bindings.as_mut() else { return Unchanged };
|
||||
Arc::make_mut(bindings).revisions.insert(seq, replacement.clone());
|
||||
lease.manifest_snapshot_generation = generation;
|
||||
if let Some(snapshot) = lease.last_manifest_snapshot.as_mut() {
|
||||
snapshot.snapshot_generation = generation;
|
||||
if let Some(publication) = snapshot.startup_revisions.as_mut() {
|
||||
if Arc::make_mut(publication).revisions.contains_key(&seq) {
|
||||
Arc::make_mut(publication).revisions.insert(seq, replacement.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = self.advance_availability_evidence(&session.proxy_session_id);
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={seq} generation={generation} old={:?} new={:?} prefix={} exposed={} outcome=replacement_bound",
|
||||
crate::safe_hls_access_lease_id(lease_id), old.id(), replacement.revision().id(),
|
||||
old.prefix_available.load(Ordering::Acquire), old.was_exposed(),
|
||||
);
|
||||
ReplacementBound
|
||||
}
|
||||
|
||||
/// An identity-checked, idempotent live -> ended transition. Removal and all
|
||||
/// existing external cleanup happen through the manager's prepared removal.
|
||||
pub(crate) fn end_revision_playback_if_current(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> Option<HlsAccessLeaseRemovalPreparation> {
|
||||
if !self.revision_recovery_is_current(lease_id, expected, session, now_ms)
|
||||
|| !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed)
|
||||
|| self.availability_evidence_can_advance().is_err()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let lease = self.by_lease_id.get_mut(lease_id)?;
|
||||
if lease.playback_mode != HlsLeasePlaybackMode::Live {
|
||||
return None;
|
||||
}
|
||||
lease.end_playback();
|
||||
lease.state = HlsAccessLeaseState::Denied;
|
||||
let _ = self.advance_availability_evidence(&session.proxy_session_id);
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=reentry_required",
|
||||
crate::safe_hls_access_lease_id(lease_id), expected.revision.revision().key.proxy_seq,
|
||||
expected.snapshot_generation, expected.revision.revision().id(),
|
||||
expected.revision.revision().prefix_available.load(Ordering::Acquire), expected.revision.revision().was_exposed(),
|
||||
);
|
||||
self.prepare_access_lease_removal(lease_id)
|
||||
}
|
||||
|
||||
pub(crate) fn revision_bound_lease_ids(&self, session: &crate::ProxySessionId, seq: u64) -> Vec<HlsAccessLeaseId> {
|
||||
self.by_lease_id
|
||||
.values()
|
||||
.filter(|lease| {
|
||||
lease.proxy_session_id == *session
|
||||
&& lease.playback_mode == HlsLeasePlaybackMode::Live
|
||||
&& lease.revision_bindings.as_ref().is_some_and(|bindings| bindings.revisions.contains_key(&seq))
|
||||
})
|
||||
.map(|lease| lease.lease_id.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -532,6 +532,7 @@ fn published_uri_rejects_a_new_attempt_before_the_first_media_request() -> std::
|
||||
}
|
||||
let mut first = manifest_snapshot(10);
|
||||
first.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 40,
|
||||
revisions: [(40, published.clone()), (41, tail.clone())].into_iter().collect(),
|
||||
@@ -545,6 +546,7 @@ fn published_uri_rejects_a_new_attempt_before_the_first_media_request() -> std::
|
||||
next.revision().prefix_available.store(8, std::sync::atomic::Ordering::Release);
|
||||
let mut retry = manifest_snapshot(11);
|
||||
retry.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 40,
|
||||
revisions: [(40, next), (41, tail)].into_iter().collect(),
|
||||
|
||||
@@ -12,6 +12,7 @@ fn retired_revision_bindings_cannot_be_rebound_after_the_hard_limit() -> std::io
|
||||
revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
let mut snapshot = manifest_snapshot(seq);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 0,
|
||||
revisions: [(seq, revision)].into_iter().collect(),
|
||||
@@ -25,6 +26,7 @@ fn retired_revision_bindings_cannot_be_rebound_after_the_hard_limit() -> std::io
|
||||
new_attempt.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
let mut old_uri = manifest_snapshot(1031);
|
||||
old_uri.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 0,
|
||||
revisions: [(0, new_attempt)].into_iter().collect(),
|
||||
@@ -49,6 +51,7 @@ fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io::
|
||||
}
|
||||
let mut snapshot = manifest_snapshot(generation + 1);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 40,
|
||||
revisions: owners,
|
||||
@@ -66,20 +69,111 @@ fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io::
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_only_accepts_the_published_head_once() {
|
||||
fn startup_claim_fixture() -> std::io::Result<(HlsAccessLeaseStore, HlsAccessLeaseId)> {
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let lease_id = HlsAccessLeaseId("claim-lease".into());
|
||||
let mut leases = HlsAccessLeaseStore::default();
|
||||
leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(10), 2000).is_committed());
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
if let Some(lease) = leases.by_lease_id.get_mut(&lease_id) {
|
||||
lease.playback_cursor.first_requested_proxy_seq = Some(41);
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let mut revisions = std::collections::BTreeMap::new();
|
||||
for seq in [40, 41] {
|
||||
let revision = store.create(session.clone(), seq, crate::SegmentRevisionKind::Raw)?;
|
||||
revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
revisions.insert(seq, revision);
|
||||
}
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2101));
|
||||
let mut snapshot = manifest_snapshot(10);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: 40,
|
||||
revisions,
|
||||
}));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2000).is_committed());
|
||||
Ok((leases, lease_id))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_accepts_the_first_requested_published_segment_once() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 42, 2100), "unpublished URI cannot consume startup");
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101), "the player may start after the head");
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2102));
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2102));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_accepts_a_retained_uri_after_the_playlist_head_moves() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let mut snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
snapshot.first_proxy_seq = 41;
|
||||
snapshot.visible_segments = Arc::from([snapshot.visible_segments[1].clone()]);
|
||||
Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).revisions.remove(&40);
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed());
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2201), "an earlier published URI is still bound");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_lease_gets_its_own_startup_claim_for_the_shared_revision() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
let other_id = HlsAccessLeaseId("other-lease".into());
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
assert!(leases.prepare_access_lease(lease(other_id.clone(), &session.0, 2000)));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &other_id, &session, snapshot, 2100).is_committed());
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101));
|
||||
assert!(leases.try_claim_progressive_startup(&other_id, 41, 2102));
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2103));
|
||||
assert!(!leases.try_claim_progressive_startup(&other_id, 40, 2103));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_cannot_be_revived_by_a_playlist_refresh_or_cursor_seek() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed());
|
||||
let _ = leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_cursor.record_request_started(100, 2201);
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2202));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_is_scoped_to_a_valid_live_lease() -> std::io::Result<()> {
|
||||
for state in [super::HlsAccessLeaseState::Denied, super::HlsAccessLeaseState::Expired] {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
leases.by_lease_id.get_mut(&lease_id).expect("lease").state = state;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
}
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 16_000));
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_mode = super::HlsLeasePlaybackMode::Ended;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_startup_claims_allow_exactly_one_request() -> std::io::Result<()> {
|
||||
let (leases, lease_id) = startup_claim_fixture()?;
|
||||
let leases = Arc::new(tokio::sync::RwLock::new(leases));
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let claim = |seq| {
|
||||
let leases = Arc::clone(&leases);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
let lease_id = lease_id.clone();
|
||||
tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
leases.write().await.try_claim_progressive_startup(&lease_id, seq, 2100)
|
||||
})
|
||||
};
|
||||
let (first, second) = tokio::join!(claim(40), claim(41));
|
||||
assert_ne!(first.expect("first task"), second.expect("second task"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -89,6 +183,7 @@ fn optimistic_binding_accepts_pending_successors_but_not_failed_or_replaced_ones
|
||||
let snapshot_with = |revisions: std::collections::BTreeMap<u64, crate::SegmentRevisionGuard>| {
|
||||
let mut snapshot = manifest_snapshot(1);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: 0,
|
||||
revisions,
|
||||
@@ -125,6 +220,7 @@ fn failed_unexposed_successor_may_be_rebound_but_exposed_bytes_never_change() ->
|
||||
let snapshot_with = |revisions: std::collections::BTreeMap<u64, crate::SegmentRevisionGuard>| {
|
||||
let mut snapshot = manifest_snapshot(1);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: 0,
|
||||
revisions,
|
||||
|
||||
@@ -51,6 +51,7 @@ mod ids;
|
||||
mod late_custom_response;
|
||||
mod lease;
|
||||
mod lifecycle;
|
||||
mod live_window;
|
||||
mod manager;
|
||||
mod manifest_acceptance;
|
||||
mod manifest_commit;
|
||||
@@ -124,7 +125,7 @@ pub mod api {
|
||||
HlsTerminalResolution,
|
||||
},
|
||||
availability_reevaluation::{HlsAvailabilityReevaluationObservation, HlsAvailabilityReevaluationRegistration},
|
||||
cache::{SegmentCacheKey, TransientObjectCacheKey},
|
||||
cache::{CachedSegmentMetadata, SegmentCacheKey, TransientObjectCacheKey},
|
||||
deadline::hls_object_body_deadline,
|
||||
gc::exec_hls_cache_gc,
|
||||
headers::{
|
||||
@@ -139,11 +140,12 @@ pub mod api {
|
||||
HlsFreshManifestRequiredReason, HlsLeaseStartupAdmissionState, HlsMediaLeaseIdentity, HlsPlaybackFamilyKey,
|
||||
},
|
||||
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
|
||||
live_window::{HlsLiveWindowPlan, HlsLiveWindowView},
|
||||
manager::{exec_hls_lifecycle, HlsMediaActivityCommitOutcome, HlsProxyManager},
|
||||
manifest_commit::{
|
||||
hls_cached_manifest_options_for_requirement, hls_committed_manifest_body_for_request,
|
||||
hls_manifest_commit_requirement, hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions,
|
||||
HlsCommittedManifestBody,
|
||||
hls_live_window_origin_allowed_for_request, hls_manifest_commit_requirement,
|
||||
hls_should_wait_for_initial_manifest_commit, HlsCachedManifestOptions, HlsCommittedManifestBody,
|
||||
},
|
||||
manifest_fetch::{LiveHlsOriginEntry, RetryPolicy},
|
||||
manifest_limits::{HlsManifestLimitViolation, MAX_HLS_MANIFEST_BYTES},
|
||||
@@ -183,7 +185,7 @@ pub mod api {
|
||||
cold_start_retry_after_seconds, maybe_trigger_origin_refresh_with_outcome, trigger_origin_refresh_sync,
|
||||
HlsManifestCommitRequirement, HlsOriginRefreshTriggerOutcome, HlsPostRefreshRuntime, OriginRefreshRequest,
|
||||
},
|
||||
renderer::RenderedManifest,
|
||||
renderer::{HlsManifestRenderer, RenderedManifest},
|
||||
resource_fetch::{
|
||||
retry_after_secs_from_ms, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsResourceFetchAttempt,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
//! One metadata-only window plan for accelerated, untagged live playlists.
|
||||
use crate::{
|
||||
media_reserve::HlsManifestCommitIdentity, HlsAccessLease, HlsAccessLeaseState, HlsSession, HlsSessionMode,
|
||||
};
|
||||
use tuliprox_core::model::StripConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HlsLiveWindowView {
|
||||
Entry,
|
||||
Active { first_published: u64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HlsLiveWindowPlan {
|
||||
pub visible: Vec<u64>,
|
||||
pub hidden_tail: Vec<u64>,
|
||||
pub effective_strip_segments: usize,
|
||||
pub origin_commits: u64,
|
||||
pub origin_work_generation: u64,
|
||||
pub source_commit_identity: Option<HlsManifestCommitIdentity>,
|
||||
origin_sequences: Vec<u64>,
|
||||
strip: StripConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HlsLiveWindowPublication {
|
||||
pub plan: HlsLiveWindowPlan,
|
||||
pub body: std::sync::Arc<str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct HlsLiveWindowDemand {
|
||||
pub(crate) issued_at_ms: u64,
|
||||
pub(crate) view: HlsLiveWindowView,
|
||||
pub(crate) published: Vec<u64>,
|
||||
}
|
||||
|
||||
impl HlsLiveWindowView {
|
||||
pub fn for_lease(lease: &HlsAccessLease) -> Self {
|
||||
if lease.state == HlsAccessLeaseState::Activated {
|
||||
if let Some(snapshot) = &lease.last_manifest_snapshot {
|
||||
return Self::Active { first_published: snapshot.first_proxy_seq };
|
||||
}
|
||||
}
|
||||
Self::Entry
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsLiveWindowPlan {
|
||||
pub fn is_current(&self, session: &HlsSession) -> bool {
|
||||
session.uses_fast_start_live_window()
|
||||
&& self.origin_commits == session.origin_manifest_commits()
|
||||
&& self.origin_work_generation == session.activity.origin_work_generation
|
||||
&& self.source_commit_identity == session.live_window_source_identity()
|
||||
&& self.origin_sequences.as_slice() == session.live_origin_sequences()
|
||||
&& self.strip == session.live_window_strip
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsSession {
|
||||
pub(crate) fn live_origin_sequences(&self) -> &[u64] {
|
||||
self.origin_window_proxy_seqs.as_deref().unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn live_window_source_identity(&self) -> Option<HlsManifestCommitIdentity> {
|
||||
if self
|
||||
.live_window_origin_work_generation
|
||||
.is_some_and(|generation| generation != self.activity.origin_work_generation)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.live_window_origin_commit_identity.or_else(|| self.last_normal_manifest_commit_identity())
|
||||
}
|
||||
|
||||
pub fn uses_fast_start_live_window(&self) -> bool {
|
||||
self.startup.is_some()
|
||||
&& matches!(self.mode, HlsSessionMode::NormalCacheTimeline)
|
||||
&& self.origin_source.archive_reference.is_none()
|
||||
&& (self.origin_window_proxy_seqs.is_some() || self.segments.is_empty())
|
||||
&& self.live_origin_sequences().iter().all(|seq| {
|
||||
self.segments
|
||||
.get(seq)
|
||||
.is_none_or(|entry| entry.is_fast_start_eligible() && !crate::is_hls_provisioning_segment(entry))
|
||||
})
|
||||
}
|
||||
|
||||
/// Entry selection uses only origin metadata. Active slides may retain cached
|
||||
/// predecessors from this view to preserve three target durations.
|
||||
pub fn plan_live_window(&self, view: HlsLiveWindowView) -> Option<HlsLiveWindowPlan> {
|
||||
if !self.uses_fast_start_live_window()
|
||||
|| self.live_origin_sequences().is_empty()
|
||||
|| self
|
||||
.live_window_origin_work_generation
|
||||
.is_some_and(|generation| generation != self.activity.origin_work_generation)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let origin = self.live_origin_sequences();
|
||||
let (visible, hidden_tail, effective_strip_segments) = match view {
|
||||
HlsLiveWindowView::Entry => {
|
||||
let strip = crate::initial_strip::initial_hls_strip_segments_for_durations(
|
||||
&self.live_window_strip,
|
||||
&self.origin_window_durations_ms,
|
||||
);
|
||||
let end = origin.len().saturating_sub(strip);
|
||||
(origin[end.saturating_sub(3)..end].to_vec(), origin[end..].to_vec(), strip)
|
||||
}
|
||||
HlsLiveWindowView::Active { first_published } => {
|
||||
let available = origin.iter().copied().filter(|seq| *seq >= first_published).collect::<Vec<_>>();
|
||||
let mut visible = available[available.len().saturating_sub(6)..].to_vec();
|
||||
crate::renderer::retain_published_predecessors_from(self, &mut visible, first_published);
|
||||
(visible, Vec::new(), 0)
|
||||
}
|
||||
};
|
||||
if visible.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(HlsLiveWindowPlan {
|
||||
visible,
|
||||
hidden_tail,
|
||||
effective_strip_segments,
|
||||
origin_commits: self.origin_manifest_commits(),
|
||||
origin_work_generation: self.activity.origin_work_generation,
|
||||
source_commit_identity: self.live_window_source_identity(),
|
||||
origin_sequences: origin.to_vec(),
|
||||
strip: self.live_window_strip.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn plan_live_window_for_lease(&mut self, lease: &HlsAccessLease) -> Option<HlsLiveWindowPlan> {
|
||||
let view = HlsLiveWindowView::for_lease(lease);
|
||||
let plan = self.plan_live_window(view)?;
|
||||
self.live_window_demands.insert(
|
||||
lease.lease_id.clone(),
|
||||
HlsLiveWindowDemand {
|
||||
issued_at_ms: lease.issued_at_ms,
|
||||
view,
|
||||
published: lease
|
||||
.last_manifest_snapshot
|
||||
.as_ref()
|
||||
.map_or_else(Vec::new, |snapshot| snapshot.visible_proxy_seqs().collect()),
|
||||
},
|
||||
);
|
||||
Some(plan)
|
||||
}
|
||||
|
||||
pub(crate) fn required_live_windows(&self) -> Vec<HlsLiveWindowPlan> {
|
||||
if self.live_window_demands.is_empty() {
|
||||
return self.plan_live_window(HlsLiveWindowView::Entry).into_iter().collect();
|
||||
}
|
||||
self.live_window_demands.values().filter_map(|demand| self.plan_live_window(demand.view)).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn live_window_heads(&self) -> Vec<u64> {
|
||||
self.required_live_windows().iter().filter_map(|plan| plan.visible.first().copied()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn record_live_window_publication(
|
||||
&mut self,
|
||||
lease_id: &crate::HlsAccessLeaseId,
|
||||
snapshot: &crate::media_reserve::HlsLeaseManifestSnapshot,
|
||||
) {
|
||||
if let Some(demand) = self.live_window_demands.get_mut(lease_id) {
|
||||
demand.published = snapshot.visible_proxy_seqs().collect();
|
||||
}
|
||||
self.longest_rendered_playlist_duration_ms =
|
||||
self.longest_rendered_playlist_duration_ms.max(snapshot.playlist_duration_ms);
|
||||
for seq in snapshot.visible_proxy_seqs() {
|
||||
if let Some(entry) = self.segments.get_mut(&seq) {
|
||||
entry.last_rendered_at_ms = Some(snapshot.delivered_at_ms);
|
||||
if let Some(identity) = entry.media_resource_identity() {
|
||||
self.published_resource_history.record(identity, seq);
|
||||
}
|
||||
if self.published_live_origin_baseline.is_none() {
|
||||
self.published_live_origin_baseline = Some(crate::session::HlsPublishedLiveOriginBaseline {
|
||||
evidence_proxy_seq: seq,
|
||||
origin_epoch: entry.origin_key.origin_epoch,
|
||||
rendered_at_ms: snapshot.delivered_at_ms,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,7 @@ pub struct HlsProxyManager {
|
||||
transient_resources: Arc<TransientResourceStore>,
|
||||
access_leases: Arc<RwLock<HlsAccessLeaseStore>>,
|
||||
lifecycle: Arc<HlsLifecycleManager>,
|
||||
lease_completion: Arc<lease_completion::HlsLeaseCompletion>,
|
||||
account_overlap_cooldowns: Arc<RwLock<HashMap<HlsAccountOverlapCooldownKey, HlsAccountOverlapCooldown>>>,
|
||||
metrics: Arc<HlsCacheMetrics>,
|
||||
qos: Arc<HlsQosRegistry>,
|
||||
@@ -153,8 +154,10 @@ mod account_overlap;
|
||||
mod activity;
|
||||
mod cleanup;
|
||||
mod config;
|
||||
mod lease_completion;
|
||||
mod lifecycle;
|
||||
mod publication;
|
||||
mod revision_recovery;
|
||||
mod terminal;
|
||||
|
||||
pub use self::lifecycle::exec_hls_lifecycle;
|
||||
@@ -162,6 +165,7 @@ pub use self::lifecycle::exec_hls_lifecycle;
|
||||
use self::lifecycle::hls_session_idle_protection_retry_at;
|
||||
#[cfg(test)]
|
||||
use self::publication::HlsRecoveryExecutionState;
|
||||
pub(crate) use self::revision_recovery::HlsLeaseRuntime;
|
||||
use self::{
|
||||
account_overlap::{HlsAccountOverlapCooldown, HlsAccountOverlapCooldownKey},
|
||||
activity::hls_key_readiness_evidence_is_current,
|
||||
|
||||
@@ -111,7 +111,7 @@ impl HlsProxyManager {
|
||||
gc.install_revision_store(&revision_store);
|
||||
segment_cache.install_revision_metrics(&metrics);
|
||||
let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default());
|
||||
Self {
|
||||
let manager = Self {
|
||||
sessions,
|
||||
segment_cache,
|
||||
segment_repair,
|
||||
@@ -135,6 +135,7 @@ impl HlsProxyManager {
|
||||
transient_resources: Arc::new(TransientResourceStore::new()),
|
||||
access_leases,
|
||||
lifecycle,
|
||||
lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()),
|
||||
account_overlap_cooldowns,
|
||||
metrics,
|
||||
qos,
|
||||
@@ -146,7 +147,9 @@ impl HlsProxyManager {
|
||||
availability_reevaluations,
|
||||
terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()),
|
||||
startup_observability: Arc::new(HlsStartupObservability::default()),
|
||||
}
|
||||
};
|
||||
manager.install_worker_revision_recovery();
|
||||
manager
|
||||
}
|
||||
|
||||
pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() }
|
||||
@@ -198,10 +201,9 @@ impl HlsProxyManager {
|
||||
self.clear_runtime_cache_state_for_cache_path_change().await;
|
||||
}
|
||||
for session in self.sessions.list_sessions().await {
|
||||
session
|
||||
.write()
|
||||
.await
|
||||
.configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth);
|
||||
let mut session = session.write().await;
|
||||
session.configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth);
|
||||
session.live_window_strip.clone_from(&runtime_config.strip);
|
||||
}
|
||||
self.segment_repair.update_config(hls_config.segment_repair.clone());
|
||||
let global_fetch_semaphore =
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
//! Lease completion belongs to the runtime, not to the HTTP future waiting for it.
|
||||
use super::{HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsProxyManager, ProxySessionId};
|
||||
use crate::{safe_proxy_session_id, sync_ext::MutexExt, HlsTerminalTailProtectionRemoval};
|
||||
use arc_swap::ArcSwapOption;
|
||||
use log::debug;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
future::Future,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use tokio::sync::{watch, Notify};
|
||||
use tuliprox_session::ActiveUserManager;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub(super) struct LeaseCompletionKey {
|
||||
pub(super) lease_id: HlsAccessLeaseId,
|
||||
pub(super) session_id: ProxySessionId,
|
||||
pub(super) issued_at_ms: u64,
|
||||
pub(super) snapshot_generation: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct HlsLeaseCompletion {
|
||||
users: ArcSwapOption<ActiveUserManager>,
|
||||
users_bound: Notify,
|
||||
jobs: Mutex<HashMap<LeaseCompletionKey, watch::Receiver<Option<bool>>>>,
|
||||
}
|
||||
|
||||
impl HlsLeaseCompletion {
|
||||
fn bind_users(&self, users: &Arc<ActiveUserManager>) {
|
||||
self.users.store(Some(Arc::clone(users)));
|
||||
self.users_bound.notify_waiters();
|
||||
}
|
||||
|
||||
pub(super) async fn users(&self) -> Arc<ActiveUserManager> {
|
||||
loop {
|
||||
let notified = self.users_bound.notified();
|
||||
if let Some(users) = self.users.load_full() {
|
||||
return users;
|
||||
}
|
||||
// Before runtime attachment the lease remains its cleanup ticket;
|
||||
// an absent accounting owner must never silently skip the release.
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run(
|
||||
self: &Arc<Self>,
|
||||
key: LeaseCompletionKey,
|
||||
completion: impl Future<Output = bool> + Send + 'static,
|
||||
) -> bool {
|
||||
let mut receiver = {
|
||||
let mut jobs = self.jobs.lock_unpoisoned();
|
||||
if let Some(receiver) = jobs.get(&key) {
|
||||
receiver.clone()
|
||||
} else {
|
||||
let (sender, receiver) = watch::channel(None);
|
||||
jobs.insert(key.clone(), receiver.clone());
|
||||
let owner = Arc::clone(self);
|
||||
// Registered and spawned before the first await. Cancelling any
|
||||
// waiter cannot cancel or erase the runtime's completion job.
|
||||
tokio::spawn(async move {
|
||||
let result = completion.await;
|
||||
owner.jobs.lock_unpoisoned().remove(&key);
|
||||
let _ = sender.send(Some(result));
|
||||
});
|
||||
receiver
|
||||
}
|
||||
};
|
||||
loop {
|
||||
if let Some(result) = *receiver.borrow_and_update() {
|
||||
return result;
|
||||
}
|
||||
if receiver.changed().await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pending_count(&self) -> usize { self.jobs.lock_unpoisoned().len() }
|
||||
}
|
||||
|
||||
impl HlsProxyManager {
|
||||
/// Attach the server's authoritative user registry before serving leases.
|
||||
pub fn bind_lease_user_accounting(&self, users: &Arc<ActiveUserManager>) {
|
||||
self.lease_completion.bind_users(users);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pending_lease_completions_for_test(&self) -> usize { self.lease_completion.pending_count() }
|
||||
}
|
||||
|
||||
impl super::HlsLeaseRuntime {
|
||||
pub(super) async fn remove_prepared_access_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
let key = LeaseCompletionKey {
|
||||
lease_id: lease_id.clone(),
|
||||
session_id: preparation.proxy_session_id.clone(),
|
||||
issued_at_ms: preparation.issued_at_ms,
|
||||
snapshot_generation: None,
|
||||
};
|
||||
let runtime = self.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
let preparation = preparation.clone();
|
||||
self.lease_completion
|
||||
.run(key, async move { runtime.complete_prepared_lease(&lease_id, &preparation).await })
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl super::HlsLeaseRuntime {
|
||||
pub(super) async fn complete_prepared_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &crate::lease::HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
// Hold the index and lease-store fences through external cleanup. A
|
||||
// replacement with the same ID must not acquire QoS/repair ownership
|
||||
// between removing the old incarnation and clearing those registries.
|
||||
// Reject stale tickets before waiting for runtime attachment.
|
||||
if !self.access_leases.read().await.prepare_access_lease_removal(lease_id).is_some_and(|current| {
|
||||
current.proxy_session_id == preparation.proxy_session_id && current.issued_at_ms == preparation.issued_at_ms
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
let users = self.lease_completion.users().await;
|
||||
let current = self.sessions.hold_current_proxy_session(&preparation.proxy_session_id).await;
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let Some(lease) = leases.response_snapshot(
|
||||
lease_id,
|
||||
&preparation.proxy_session_id,
|
||||
tuliprox_core::utils::current_time_millis(),
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
if lease.issued_at_ms != preparation.issued_at_ms {
|
||||
return false;
|
||||
}
|
||||
users.release_session_streams_and_counted_reservation(&lease.username, &lease.user_session_token).await;
|
||||
self.terminal_pending.cancel_lease(lease_id);
|
||||
self.terminal_commit_retries.cancel_lease(lease_id);
|
||||
if let Some(current) = current.as_ref() {
|
||||
let session = current.session();
|
||||
let mut session = session.write().await;
|
||||
if session
|
||||
.live_window_demands
|
||||
.get(lease_id)
|
||||
.is_some_and(|demand| demand.issued_at_ms == preparation.issued_at_ms)
|
||||
{
|
||||
session.live_window_demands.remove(lease_id);
|
||||
}
|
||||
let finalized_manifest_released =
|
||||
session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms);
|
||||
if finalized_manifest_released {
|
||||
debug!(
|
||||
"HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
if let Some(generation) = preparation.terminal_protection_generation {
|
||||
let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation);
|
||||
match removal {
|
||||
HlsTerminalTailProtectionRemoval::Removed => {
|
||||
debug!(
|
||||
"HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0
|
||||
);
|
||||
}
|
||||
HlsTerminalTailProtectionRemoval::Missing => {}
|
||||
HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => {
|
||||
debug!(
|
||||
"HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0,
|
||||
actual.0
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if session.remove_terminal_tail_protection(lease_id).is_some() {
|
||||
debug!(
|
||||
"HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
self.standalone_custom_access.remove(lease_id);
|
||||
self.segment_repair.remove_access_lease_window(lease_id).await;
|
||||
self.startup_observability.remove_access_lease(lease_id);
|
||||
self.qos.remove_access_lease(lease_id).await;
|
||||
// Keep the exact lease present until all release steps have finished.
|
||||
leases.remove_access_lease_if_preparation_matches(lease_id, preparation).is_some()
|
||||
}
|
||||
}
|
||||
@@ -51,8 +51,6 @@ impl HlsProxyManager {
|
||||
}
|
||||
|
||||
pub(super) async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) {
|
||||
self.terminal_pending.cancel_lease(lease_id);
|
||||
self.terminal_commit_retries.cancel_lease(lease_id);
|
||||
let Some(preparation) = self.access_leases.read().await.prepare_access_lease_removal(lease_id) else {
|
||||
return;
|
||||
};
|
||||
@@ -65,65 +63,15 @@ impl HlsProxyManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes one exact lease incarnation before releasing any media
|
||||
/// protection owned by it. A stale preparation is intentionally
|
||||
/// non-destructive because the replacement lease may reuse the same ID.
|
||||
/// Complete one exact lease incarnation in the shared, request-independent
|
||||
/// runtime. The lease is retired after accounting and media protection are
|
||||
/// released; stale preparations cannot affect a replacement with the same ID.
|
||||
pub(super) async fn remove_prepared_access_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
let removed =
|
||||
self.access_leases.write().await.remove_access_lease_if_preparation_matches(lease_id, preparation);
|
||||
if removed.is_none() {
|
||||
return false;
|
||||
}
|
||||
if let Some(session) = self.sessions.get_by_proxy_session_id(&preparation.proxy_session_id).await {
|
||||
let mut session = session.write().await;
|
||||
let finalized_manifest_released =
|
||||
session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms);
|
||||
if finalized_manifest_released {
|
||||
debug!(
|
||||
"HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
if let Some(generation) = preparation.terminal_protection_generation {
|
||||
let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation);
|
||||
match removal {
|
||||
HlsTerminalTailProtectionRemoval::Removed => {
|
||||
debug!(
|
||||
"HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0
|
||||
);
|
||||
}
|
||||
HlsTerminalTailProtectionRemoval::Missing => {}
|
||||
HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => {
|
||||
debug!(
|
||||
"HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0,
|
||||
actual.0
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if session.remove_terminal_tail_protection(lease_id).is_some() {
|
||||
debug!(
|
||||
"HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
self.standalone_custom_access.remove(lease_id);
|
||||
self.segment_repair.remove_access_lease_window(lease_id).await;
|
||||
self.startup_observability.remove_access_lease(lease_id);
|
||||
self.qos.remove_access_lease(lease_id).await;
|
||||
true
|
||||
self.revision_recovery_runtime().remove_prepared_access_lease(lease_id, preparation).await
|
||||
}
|
||||
|
||||
pub(super) async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) {
|
||||
@@ -205,6 +153,7 @@ impl HlsProxyManager {
|
||||
event: HlsLifecycleEvent,
|
||||
now_ms: u64,
|
||||
) {
|
||||
self.bind_lease_user_accounting(active_users);
|
||||
if !self.is_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -214,7 +163,8 @@ impl HlsProxyManager {
|
||||
let mut should_sync_session = false;
|
||||
if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await {
|
||||
should_sync_session = true;
|
||||
if let Some(release) = &snapshot.idle_release {
|
||||
let ending = matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied);
|
||||
if let Some(release) = snapshot.idle_release.as_ref().filter(|_| !ending) {
|
||||
active_users
|
||||
.release_session_streams_and_counted_reservation(
|
||||
&release.username,
|
||||
@@ -228,7 +178,7 @@ impl HlsProxyManager {
|
||||
super::super::safe_user_session_token(&release.user_session_token)
|
||||
);
|
||||
}
|
||||
if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) {
|
||||
if ending {
|
||||
self.remove_access_lease(&snapshot.lease_id).await;
|
||||
debug!(
|
||||
"HLS access lease removed: lease={} proxy_session={} state={}",
|
||||
@@ -385,6 +335,7 @@ impl HlsProxyManager {
|
||||
}
|
||||
|
||||
pub fn exec_hls_lifecycle(ctx: &HlsCtx, cancel_token: &CancellationToken) {
|
||||
ctx.attach_lease_user_accounting();
|
||||
let hls_proxy = Arc::clone(&ctx.hls_proxy);
|
||||
let active_users = Arc::clone(&ctx.active_users);
|
||||
let active_provider = Arc::clone(&ctx.active_provider);
|
||||
|
||||
@@ -160,7 +160,7 @@ impl HlsProxyManager {
|
||||
gc.install_revision_store(&revision_store);
|
||||
segment_cache.install_revision_metrics(&metrics);
|
||||
let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default());
|
||||
Self {
|
||||
let manager = Self {
|
||||
sessions,
|
||||
segment_cache,
|
||||
segment_repair,
|
||||
@@ -184,6 +184,7 @@ impl HlsProxyManager {
|
||||
transient_resources: Arc::new(TransientResourceStore::new()),
|
||||
access_leases,
|
||||
lifecycle,
|
||||
lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()),
|
||||
account_overlap_cooldowns,
|
||||
metrics,
|
||||
qos,
|
||||
@@ -195,7 +196,9 @@ impl HlsProxyManager {
|
||||
availability_reevaluations,
|
||||
terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()),
|
||||
startup_observability: Arc::new(HlsStartupObservability::default()),
|
||||
}
|
||||
};
|
||||
manager.install_worker_revision_recovery();
|
||||
manager
|
||||
}
|
||||
|
||||
pub fn sessions(&self) -> &Arc<HlsSessionStore> { &self.sessions }
|
||||
@@ -475,6 +478,18 @@ impl HlsProxyManager {
|
||||
published_transient_resource_ids: HlsPublishedTransientResourceIds,
|
||||
now_ms: u64,
|
||||
) -> HlsLeaseManifestPublicationOutcome {
|
||||
if snapshot.startup_revisions.is_some() {
|
||||
return self
|
||||
.commit_recovered_startup_manifest(
|
||||
lease_id,
|
||||
proxy_session_id,
|
||||
expected,
|
||||
snapshot,
|
||||
published_transient_resource_ids,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation;
|
||||
let lease_issued_at_ms = expected.lease_issued_at_ms();
|
||||
let Some(finalized_manifest_generation) = finalized_manifest_generation else {
|
||||
@@ -909,7 +924,11 @@ impl HlsProxyManager {
|
||||
}
|
||||
}
|
||||
self.schedule_session_idle_for_handle(&session).await;
|
||||
session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth);
|
||||
{
|
||||
let mut session = session.write().await;
|
||||
session.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth);
|
||||
session.live_window_strip = self.strip();
|
||||
}
|
||||
(session, outcome)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
use crate::{
|
||||
lease::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome},
|
||||
HlsAccessLeaseId, HlsMediaLeaseIdentity, HlsProxyManager, ProxySessionId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Manager-owned resources used by fetch completion, even when an origin I/O
|
||||
/// context is absent. Sharing this handle avoids a second lease cleanup path.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct HlsLeaseRuntime {
|
||||
pub(super) sessions: Arc<crate::HlsSessionStore>,
|
||||
pub(super) lease_completion: Arc<super::lease_completion::HlsLeaseCompletion>,
|
||||
pub(super) access_leases: Arc<tokio::sync::RwLock<crate::HlsAccessLeaseStore>>,
|
||||
pub(super) standalone_custom_access: Arc<crate::runtime_custom_tail::HlsStandaloneCustomAccessStore>,
|
||||
pub(super) segment_repair: Arc<crate::HlsSegmentRepairManager>,
|
||||
pub(super) startup_observability: Arc<crate::HlsStartupObservability>,
|
||||
pub(super) qos: Arc<crate::HlsQosRegistry>,
|
||||
pub(super) terminal_pending: Arc<crate::terminal_pending::HlsTerminalPendingCoordinator>,
|
||||
pub(super) terminal_commit_retries: Arc<crate::terminal_commit::HlsTerminalCommitRetryCoordinator>,
|
||||
}
|
||||
|
||||
impl HlsProxyManager {
|
||||
pub(super) fn revision_recovery_runtime(&self) -> HlsLeaseRuntime {
|
||||
HlsLeaseRuntime {
|
||||
sessions: Arc::clone(&self.sessions),
|
||||
lease_completion: Arc::clone(&self.lease_completion),
|
||||
access_leases: Arc::clone(&self.access_leases),
|
||||
standalone_custom_access: Arc::clone(&self.standalone_custom_access),
|
||||
segment_repair: Arc::clone(&self.segment_repair),
|
||||
startup_observability: Arc::clone(&self.startup_observability),
|
||||
qos: Arc::clone(&self.qos),
|
||||
terminal_pending: Arc::clone(&self.terminal_pending),
|
||||
terminal_commit_retries: Arc::clone(&self.terminal_commit_retries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn install_worker_revision_recovery(&self) {
|
||||
let _ = self.segment_worker_pool.revision_recovery.set(Arc::new(self.revision_recovery_runtime()));
|
||||
}
|
||||
|
||||
pub(super) async fn commit_recovered_startup_manifest(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
expected: crate::lease::HlsLeaseManifestPublicationGuard,
|
||||
mut snapshot: crate::media_reserve::HlsLeaseManifestSnapshot,
|
||||
resources: crate::HlsPublishedTransientResourceIds,
|
||||
now_ms: u64,
|
||||
) -> crate::lease::HlsLeaseManifestPublicationOutcome {
|
||||
use crate::lease::{
|
||||
HlsLeaseManifestPublicationOutcome as Outcome, HlsLeaseManifestPublicationRejectReason as Reject,
|
||||
};
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else {
|
||||
return Outcome::Rejected(Reject::SessionMismatch);
|
||||
};
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let mut session = current.session().write().await;
|
||||
if !leases.media_identity_is_current(lease_id, session_id, expected.media_identity(), now_ms)
|
||||
|| session.is_gc_marked_for_removal()
|
||||
{
|
||||
return Outcome::Rejected(Reject::LeaseUnavailable);
|
||||
}
|
||||
if snapshot.startup_revisions.as_ref().is_some_and(|publication| publication.window_plan.is_some()) {
|
||||
session.live_window_strip = self.strip();
|
||||
}
|
||||
let retained = now_ms <= snapshot.delivered_at_ms.saturating_add(snapshot.playlist_duration_ms)
|
||||
&& leases
|
||||
.response_snapshot(lease_id, session_id, now_ms)
|
||||
.and_then(|lease| lease.last_manifest_snapshot)
|
||||
.as_ref()
|
||||
== Some(&snapshot);
|
||||
if let Some(publication) = snapshot.startup_revisions.as_mut() {
|
||||
if let Some(window) = &publication.window_plan {
|
||||
let plan = &window.plan;
|
||||
let view_matches = leases.response_snapshot(lease_id, session_id, now_ms).is_some_and(|lease| {
|
||||
session.plan_live_window(crate::live_window::HlsLiveWindowView::for_lease(&lease)).as_ref()
|
||||
== Some(plan)
|
||||
});
|
||||
if (!plan.is_current(&session) || !view_matches) && !retained {
|
||||
return Outcome::Rejected(Reject::ManifestGenerationUnavailable);
|
||||
}
|
||||
}
|
||||
for (seq, revision) in &mut std::sync::Arc::make_mut(publication).revisions {
|
||||
if let Some(guard) = leases.revision_recovery_guard(lease_id, *seq) {
|
||||
match leases.recover_revision_binding(lease_id, &guard, &session, now_ms) {
|
||||
HlsRevisionRecoveryOutcome::ReentryRequired => {
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await;
|
||||
return Outcome::Rejected(Reject::LeaseUnavailable);
|
||||
}
|
||||
HlsRevisionRecoveryOutcome::RetryPending => return Outcome::Rejected(Reject::RevisionConflict),
|
||||
HlsRevisionRecoveryOutcome::Unchanged | HlsRevisionRecoveryOutcome::ReplacementBound => {}
|
||||
}
|
||||
if let Some(bound) = leases.published_segment_revision(lease_id, *seq) {
|
||||
if revision.revision().id() < bound.revision().id()
|
||||
&& matches!(revision.revision().state(), crate::SegmentRevisionState::Failed)
|
||||
{
|
||||
*revision = bound;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Publication cannot introduce a revision left by a retired
|
||||
// fetch, nor use a caller-supplied candidate outside startup.
|
||||
if session.startup.as_ref().and_then(|startup| startup.revisions.get(seq)) != Some(revision)
|
||||
&& leases.published_segment_revision(lease_id, *seq).as_ref() != Some(revision)
|
||||
{
|
||||
return Outcome::Rejected(Reject::RevisionConflict);
|
||||
}
|
||||
}
|
||||
}
|
||||
let published = snapshot
|
||||
.startup_revisions
|
||||
.as_ref()
|
||||
.and_then(|publication| publication.window_plan.as_ref())
|
||||
.map(|_| snapshot.clone());
|
||||
let outcome = leases
|
||||
.commit_manifest_publication_with_resources(lease_id, session_id, expected, snapshot, resources, now_ms);
|
||||
if outcome.is_committed() {
|
||||
if let Some(snapshot) = published {
|
||||
session.record_live_window_publication(lease_id, &snapshot);
|
||||
}
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
/// Resolves one published URI against the exact current session and lease.
|
||||
/// The index -> lease store -> session order matches publication/cutover.
|
||||
pub(crate) async fn recover_access_lease_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
identity: HlsMediaLeaseIdentity,
|
||||
seq: u64,
|
||||
now_ms: u64,
|
||||
) -> HlsRevisionRecoveryOutcome {
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
};
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
if !leases.media_identity_is_current(lease_id, session_id, identity, now_ms) {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
}
|
||||
let Some(guard) = leases.revision_recovery_guard(lease_id, seq) else {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
};
|
||||
let outcome = leases.recover_revision_binding(lease_id, &guard, &session, now_ms);
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
if outcome == HlsRevisionRecoveryOutcome::ReentryRequired {
|
||||
self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await;
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
pub(crate) async fn end_access_lease_for_failed_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
self.revision_recovery_runtime().end_access_lease_for_failed_revision(lease_id, session_id, guard, now_ms).await
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsLeaseRuntime {
|
||||
/// Retire only the observed binding/snapshot/lease incarnation. Repeated
|
||||
/// callers and completions from older snapshots are intentionally harmless.
|
||||
pub(crate) async fn end_access_lease_for_failed_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
let (issued_at_ms, snapshot_generation) = guard.completion_identity();
|
||||
let key = super::lease_completion::LeaseCompletionKey {
|
||||
lease_id: lease_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
issued_at_ms,
|
||||
snapshot_generation: Some(snapshot_generation),
|
||||
};
|
||||
let runtime = self.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
let session_id = session_id.clone();
|
||||
let guard = guard.clone();
|
||||
self.lease_completion
|
||||
.run(key, async move { runtime.end_failed_revision_owned(&lease_id, &session_id, &guard, now_ms).await })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn end_failed_revision_owned(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
let _users = self.lease_completion.users().await;
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return false };
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
let ended_at_ms = tuliprox_core::utils::current_time_millis().max(now_ms);
|
||||
let preparation = leases.end_revision_playback_if_current(lease_id, guard, &session, ended_at_ms);
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
let Some(preparation) = preparation else { return false };
|
||||
self.complete_prepared_lease(lease_id, &preparation).await
|
||||
}
|
||||
|
||||
pub(crate) async fn recover_segment_leases_after_fetch(
|
||||
&self,
|
||||
session_id: &ProxySessionId,
|
||||
seq: u64,
|
||||
origin_generation: u64,
|
||||
now_ms: u64,
|
||||
) {
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return };
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
if session.activity.origin_work_generation != origin_generation || session.is_gc_marked_for_removal() {
|
||||
return;
|
||||
}
|
||||
let ids = leases.revision_bound_lease_ids(session_id, seq);
|
||||
let mut endings = Vec::new();
|
||||
for id in ids {
|
||||
if let Some(guard) = leases.revision_recovery_guard(&id, seq) {
|
||||
if leases.recover_revision_binding(&id, &guard, &session, now_ms)
|
||||
== HlsRevisionRecoveryOutcome::ReentryRequired
|
||||
{
|
||||
endings.push((id, guard));
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
for (id, guard) in endings {
|
||||
self.end_access_lease_for_failed_revision(&id, session_id, &guard, now_ms).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -194,6 +194,7 @@ async fn finalized_signed_url_generations_follow_published_lease_lifetime() {
|
||||
.await
|
||||
.expect("expired lease remains available for lifecycle cleanup");
|
||||
assert_eq!(expired_a.state, HlsAccessLeaseState::Expired);
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_a_id).await;
|
||||
let mut session = session.write().await;
|
||||
assert_eq!(session.transient.finalized_manifest_generation_count(), 1);
|
||||
@@ -309,6 +310,7 @@ async fn same_lease_retains_every_published_finalized_generation_until_removal()
|
||||
.is_some());
|
||||
}
|
||||
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
let session = session.read().await;
|
||||
assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 0);
|
||||
@@ -434,6 +436,7 @@ async fn lease_reconciliation_cannot_resurrect_a_concurrently_removed_binding()
|
||||
)
|
||||
};
|
||||
wait_until_access_lease_store_is_write_locked(&manager).await;
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
let removal_task = {
|
||||
let manager = Arc::clone(&manager);
|
||||
let lease_id = lease_id.clone();
|
||||
|
||||
@@ -41,6 +41,7 @@ async fn update_config_applies_hls_runtime_settings_to_existing_manager() {
|
||||
assert_eq!(manager.strip().mode, HlsStripMode::Seconds);
|
||||
assert_eq!(manager.strip().value, 7);
|
||||
assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4);
|
||||
assert_eq!(session.read().await.live_window_strip, manager.strip());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1109,6 +1109,7 @@ async fn resource_denial_does_not_destroy_existing_terminal_tail() {
|
||||
);
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
assert_eq!(
|
||||
@@ -1148,6 +1149,7 @@ async fn terminal_tail_protection_is_retained_until_bounded_lease_removal() {
|
||||
);
|
||||
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
}
|
||||
@@ -1188,6 +1190,7 @@ async fn terminal_unavailable_protection_is_retained_until_bounded_lease_removal
|
||||
assert_eq!(denied.state, HlsAccessLeaseState::Denied);
|
||||
assert!(matches!(denied.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
}
|
||||
|
||||
@@ -112,6 +112,35 @@ pub fn hls_committed_manifest_body_for_request(
|
||||
})
|
||||
}
|
||||
|
||||
/// The accelerated lease view is rendered from an origin commit, independently
|
||||
/// of whether the shared session's older window can still be rendered.
|
||||
pub fn hls_live_window_origin_allowed_for_request(
|
||||
session: &HlsSession,
|
||||
options: HlsCachedManifestOptions,
|
||||
started_at_ms: u64,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
let Some(identity) = session.live_window_source_identity() else { return false };
|
||||
let duration = session.origin_window_durations_ms.iter().fold(0_u64, |sum, duration| sum.saturating_add(*duration));
|
||||
let candidate = HlsCommittedManifestCandidate {
|
||||
body: HlsCommittedManifestBody::Normal(String::new()),
|
||||
rendered_at_ms: identity.rendered_at_ms(),
|
||||
valid_until_ms: Some(identity.rendered_at_ms().saturating_add(duration)),
|
||||
finalized: false,
|
||||
};
|
||||
session.uses_fast_start_live_window()
|
||||
&& can_serve_committed_manifest(
|
||||
session,
|
||||
&candidate,
|
||||
session.account_binding_protection(now_ms),
|
||||
options.policy,
|
||||
candidate.rendered_at_ms >= started_at_ms,
|
||||
now_ms,
|
||||
)
|
||||
&& manifest_rendered_after_required_boundary(Some(candidate.rendered_at_ms), options)
|
||||
&& options.after_origin_manifest_commits.is_none_or(|commits| session.origin_manifest_commits() > commits)
|
||||
}
|
||||
|
||||
pub fn hls_should_wait_for_initial_manifest_commit(
|
||||
session: &HlsSession,
|
||||
selected_committed_body: bool,
|
||||
|
||||
@@ -15,6 +15,12 @@ use std::sync::Arc;
|
||||
|
||||
/// Exact manifest delivery from which lease-local playback evidence is derived.
|
||||
pub enum HlsLeaseManifestSnapshotInput<'a> {
|
||||
FastStartWindow {
|
||||
session: &'a HlsSession,
|
||||
plan: &'a crate::live_window::HlsLiveWindowPlan,
|
||||
rendered: &'a crate::renderer::RenderedManifest,
|
||||
materialized_body: &'a str,
|
||||
},
|
||||
NormalCacheTimeline {
|
||||
session: &'a HlsSession,
|
||||
committed_body: &'a str,
|
||||
@@ -91,6 +97,12 @@ pub fn derive_hls_lease_manifest_snapshot(
|
||||
delivered_at_ms: u64,
|
||||
) -> Result<Option<HlsLeaseManifestSnapshot>, HlsManifestLimitViolation> {
|
||||
match input {
|
||||
HlsLeaseManifestSnapshotInput::FastStartWindow { session, plan, rendered, materialized_body } => {
|
||||
if !plan.is_current(session) || rendered.segment_proxy_seqs != plan.visible {
|
||||
return Ok(None);
|
||||
}
|
||||
derive_rendered_snapshot(session, rendered, materialized_body, 0, delivered_at_ms, Some((*plan).clone()))
|
||||
}
|
||||
HlsLeaseManifestSnapshotInput::NormalCacheTimeline {
|
||||
session,
|
||||
committed_body,
|
||||
@@ -155,7 +167,22 @@ fn derive_normal_snapshot(
|
||||
if rendered.body != committed_body {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(source_commit_identity) = session.last_normal_manifest_commit_identity() else {
|
||||
derive_rendered_snapshot(session, rendered, materialized_body, stripped_tail_segments, delivered_at_ms, None)
|
||||
}
|
||||
|
||||
fn derive_rendered_snapshot(
|
||||
session: &HlsSession,
|
||||
rendered: &crate::renderer::RenderedManifest,
|
||||
materialized_body: &str,
|
||||
stripped_tail_segments: usize,
|
||||
delivered_at_ms: u64,
|
||||
window_plan: Option<crate::live_window::HlsLiveWindowPlan>,
|
||||
) -> Result<Option<HlsLeaseManifestSnapshot>, HlsManifestLimitViolation> {
|
||||
let Some(source_commit_identity) = window_plan
|
||||
.as_ref()
|
||||
.and_then(|plan| plan.source_commit_identity)
|
||||
.or_else(|| session.last_normal_manifest_commit_identity())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(parsed) = parse_lease_manifest(materialized_body)? else {
|
||||
@@ -210,6 +237,10 @@ fn derive_normal_snapshot(
|
||||
.map(|seq| Some((*seq, startup.revisions.get(seq)?.clone())))
|
||||
.collect::<Option<std::collections::BTreeMap<_, _>>>()?;
|
||||
snapshot.startup_revisions = Some(Arc::new(super::HlsManifestRevisions {
|
||||
window_plan: window_plan.map(|plan| crate::live_window::HlsLiveWindowPublication {
|
||||
plan,
|
||||
body: Arc::from(materialized_body),
|
||||
}),
|
||||
mode: startup.config.mode,
|
||||
retained_start_seq: session.segments.first_key_value().map_or(first_proxy_seq, |(seq, _)| *seq),
|
||||
revisions,
|
||||
|
||||
@@ -76,7 +76,6 @@ impl HlsManifestCommitIdentity {
|
||||
|
||||
pub const fn commit_generation(self) -> u64 { self.commit_generation }
|
||||
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub const fn rendered_at_ms(self) -> u64 { self.rendered_at_ms }
|
||||
}
|
||||
|
||||
@@ -1010,6 +1009,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::FirstReady,
|
||||
retained_start_seq: 0,
|
||||
revisions: owners,
|
||||
@@ -1055,6 +1055,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Progressive,
|
||||
retained_start_seq: 0,
|
||||
revisions: owners,
|
||||
@@ -1092,6 +1093,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: 0,
|
||||
revisions: owners,
|
||||
|
||||
@@ -204,6 +204,50 @@ impl HlsSession {
|
||||
if self.is_gc_marked_for_removal() {
|
||||
return report;
|
||||
}
|
||||
if self.uses_fast_start_live_window() {
|
||||
let plans = self.required_live_windows();
|
||||
let required = plans.iter().flat_map(|plan| plan.visible.iter().copied()).collect::<BTreeSet<_>>();
|
||||
let obsolete = self
|
||||
.segment_prefetch_queue
|
||||
.proxy_seqs()
|
||||
.into_iter()
|
||||
.filter(|seq| {
|
||||
!required.contains(seq)
|
||||
&& self.segments.get(seq).is_some_and(|entry| {
|
||||
matches!(
|
||||
entry.status,
|
||||
SegmentCacheStatus::Queued { priority: SegmentFetchPriority::RenderWindow, .. }
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for seq in obsolete {
|
||||
self.segment_prefetch_queue.remove(seq);
|
||||
if let Some(entry) = self.segments.get_mut(&seq) {
|
||||
entry.status = SegmentCacheStatus::Discovered;
|
||||
}
|
||||
}
|
||||
for seq in required {
|
||||
if self.queue_segment_fetch_candidate(seq, SegmentFetchPriority::RenderWindow, now_ms) {
|
||||
report.render_window_queued += 1;
|
||||
}
|
||||
}
|
||||
let reserve = plans.iter().flat_map(|plan| plan.hidden_tail.iter().copied()).collect::<BTreeSet<_>>();
|
||||
for seq in reserve {
|
||||
if allow_prefetch {
|
||||
if self.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Prefetch, now_ms) {
|
||||
report.prefetch_queued += 1;
|
||||
}
|
||||
} else if self
|
||||
.segments
|
||||
.get(&seq)
|
||||
.is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Discovered))
|
||||
{
|
||||
report.prefetch_skipped += 1;
|
||||
}
|
||||
}
|
||||
return report;
|
||||
}
|
||||
let Some(tail_proxy_seq) = self.publishable_origin_tail_proxy_seq else {
|
||||
return report;
|
||||
};
|
||||
|
||||
@@ -895,6 +895,7 @@ fn commit_normal_manifest(
|
||||
session.record_origin_manifest_commit();
|
||||
session.transient.upsert_resources(key_resources);
|
||||
session.initial_prefetch_gap_segments = initial_prefetch_gap_segments;
|
||||
session.record_live_window_origin_commit(rendered_at_ms);
|
||||
if provisioning_handoff {
|
||||
limit_publishable_normal_provisioning_handoff_tail(session, &request.strip, manifest.segments.len());
|
||||
}
|
||||
|
||||
@@ -92,3 +92,116 @@ use self::support::{
|
||||
StaleSwitchGeneration, TestOriginServer, CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, SWITCH_MAP_BODY,
|
||||
SWITCH_SEGMENT_BODY,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(
|
||||
clippy::too_many_lines,
|
||||
reason = "Keeps the degraded admission and unchanged-success counterprobe in one scenario."
|
||||
)]
|
||||
async fn retryable_manifest_failure_keeps_optimistic_admission_degraded_until_media_progress() -> std::io::Result<()> {
|
||||
use crate::{
|
||||
media_reserve::{
|
||||
evaluate_startup_admission, HlsReadyTimelineSnapshot, HlsStartupAdmissionDecision,
|
||||
HlsStartupAdmissionInput, HlsStartupAdmissionOriginState,
|
||||
},
|
||||
origin_progress::HlsOriginPathCondition,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
let session = test_session();
|
||||
let context = HlsOriginManifestFetchContext {
|
||||
app_config: test_app_config(),
|
||||
session: Arc::clone(&session),
|
||||
availability_runtime: None,
|
||||
origin_entry: LiveHlsOriginEntry::parse("http://127.0.0.1:9/index.m3u8").unwrap(),
|
||||
headers: axum::http::HeaderMap::new(),
|
||||
client: reqwest::Client::new(),
|
||||
no_redirect_client: reqwest::Client::new(),
|
||||
use_manual_redirects: false,
|
||||
origin_manifest_timeout_ms: 1000,
|
||||
manifest_recovery_burst: tuliprox_core::model::HlsManifestRecoveryBurstConfig::default(),
|
||||
retry_policy: RetryPolicy::default(),
|
||||
recovery_timing_policy: test_recovery_timing_policy(100),
|
||||
acceptance_timing_seed: None,
|
||||
};
|
||||
let manifest = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4,\n0.ts\n#EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n");
|
||||
let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(origin) =
|
||||
tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(
|
||||
&manifest.body,
|
||||
&manifest.final_manifest_url,
|
||||
)
|
||||
else {
|
||||
panic!("normal");
|
||||
};
|
||||
let snapshot = {
|
||||
let mut session = session.write().await;
|
||||
session.apply_origin_manifest(&origin).unwrap();
|
||||
session.origin_control.record_media_progress(10, 4000);
|
||||
let config = tuliprox_core::model::HlsStartupConfig {
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(crate::SegmentRevisionStore::default());
|
||||
let mut revisions = std::collections::BTreeMap::new();
|
||||
for seq in 0..3 {
|
||||
revisions
|
||||
.insert(seq, store.create(session.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?);
|
||||
}
|
||||
revisions[&0].revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
session.startup = Some(crate::HlsSessionStartup {
|
||||
config: config.clone(),
|
||||
first_data_timeout_ms: 10000,
|
||||
worker: std::sync::Weak::new(),
|
||||
access_leases: Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())),
|
||||
revisions,
|
||||
policy_fixed: true,
|
||||
store,
|
||||
budget: crate::ProgressiveBudgetManager::new(config),
|
||||
});
|
||||
let rendered = session.render_and_store_manifest(10).unwrap();
|
||||
crate::api::derive_hls_lease_manifest_snapshot(
|
||||
&crate::api::HlsLeaseManifestSnapshotInput::NormalCacheTimeline {
|
||||
session: &session,
|
||||
committed_body: &rendered.body,
|
||||
materialized_body: &rendered.body,
|
||||
stripped_tail_segments: 0,
|
||||
},
|
||||
10,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
};
|
||||
let evaluate = |origin_state| {
|
||||
evaluate_startup_admission(HlsStartupAdmissionInput {
|
||||
manifest: &snapshot,
|
||||
ready_timeline: &HlsReadyTimelineSnapshot { units: Arc::from([]) },
|
||||
origin_state,
|
||||
recovery_trigger_budget: crate::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0),
|
||||
})
|
||||
.decision
|
||||
};
|
||||
assert_eq!(evaluate(HlsStartupAdmissionOriginState::Healthy), HlsStartupAdmissionDecision::Admit);
|
||||
let generation = session.read().await.origin_control.progress_generation;
|
||||
crate::availability::record_retryable_manifest_attempt_failure(
|
||||
&context,
|
||||
generation,
|
||||
&OriginManifestFetchError::RetryableStatus(axum::http::StatusCode::BAD_GATEWAY, None),
|
||||
11,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(session.read().await.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure);
|
||||
assert!(matches!(evaluate(HlsStartupAdmissionOriginState::Degraded), HlsStartupAdmissionDecision::Reject(_)));
|
||||
{
|
||||
let mut session = session.write().await;
|
||||
let timing = build_manifest_refresh_timing(Some(4000), Some(4000), HlsManifestProgress::Unchanged);
|
||||
record_committed_manifest_media_progress(
|
||||
&mut session.origin_control,
|
||||
HlsManifestCommitProgressEvidence::CacheTimeline(timing),
|
||||
Some(4),
|
||||
12,
|
||||
);
|
||||
assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure);
|
||||
session.origin_control.record_media_progress(13, 4000);
|
||||
assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::ProgressExpected);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -91,7 +91,43 @@ enum RenderTailPolicy {
|
||||
}
|
||||
|
||||
impl HlsManifestRenderer {
|
||||
pub fn render_live_window(
|
||||
session: &HlsSession,
|
||||
plan: &crate::live_window::HlsLiveWindowPlan,
|
||||
rendered_at_ms: u64,
|
||||
) -> Result<RenderedManifest, RenderError> {
|
||||
if !plan.is_current(session) {
|
||||
return Err(RenderError::InvalidState);
|
||||
}
|
||||
let minimum = if session.allows_short_origin_windows() { 1 } else { MIN_VISIBLE_SEGMENTS };
|
||||
let startup = session.startup.as_ref().ok_or(RenderError::InvalidState)?;
|
||||
let head = plan.visible.first().ok_or(RenderError::NoRenderableWindow)?;
|
||||
if plan.visible.len() < minimum
|
||||
|| !startup
|
||||
.revisions
|
||||
.get(head)
|
||||
.is_some_and(|revision| revision.revision().is_startup_head_ready(startup.config.mode))
|
||||
|| !plan
|
||||
.visible
|
||||
.iter()
|
||||
.all(|seq| session.segments.get(seq).is_some_and(|entry| is_renderable(entry, session)))
|
||||
{
|
||||
return Err(RenderError::NoRenderableWindow);
|
||||
}
|
||||
render_window(session, &plan.visible, plan.effective_strip_segments, rendered_at_ms)
|
||||
}
|
||||
|
||||
pub fn render(session: &HlsSession, rendered_at_ms: u64) -> Result<RenderedManifest, RenderError> {
|
||||
if session.uses_fast_start_live_window() {
|
||||
let view = session
|
||||
.last_rendered_manifest
|
||||
.as_ref()
|
||||
.map_or(crate::live_window::HlsLiveWindowView::Entry, |previous| {
|
||||
crate::live_window::HlsLiveWindowView::Active { first_published: previous.first_proxy_seq }
|
||||
});
|
||||
let plan = session.plan_live_window(view).ok_or(RenderError::NoRenderableWindow)?;
|
||||
return Self::render_live_window(session, &plan, rendered_at_ms);
|
||||
}
|
||||
for render_gap_segments in 0..=session.render_policy.initial_render_gap_segments {
|
||||
let Some(window) = select_window(session, render_gap_segments, RenderTailPolicy::TruncateUnavailableTail)
|
||||
else {
|
||||
@@ -196,6 +232,14 @@ impl HlsSession {
|
||||
}
|
||||
|
||||
pub fn renderer_candidate_window_proxy_seqs(session: &HlsSession) -> Vec<u64> {
|
||||
if session.uses_fast_start_live_window() {
|
||||
return session
|
||||
.required_live_windows()
|
||||
.into_iter()
|
||||
.flat_map(|plan| plan.visible)
|
||||
.chain(session.live_window_demands.values().flat_map(|demand| demand.published.iter().copied()))
|
||||
.collect();
|
||||
}
|
||||
if session.startup.is_some() {
|
||||
let Some((head, tail)) =
|
||||
session.publishable_origin_head_proxy_seq.zip(session.publishable_origin_tail_proxy_seq)
|
||||
@@ -352,6 +396,14 @@ fn retain_published_predecessors(session: &HlsSession, window: &mut Vec<u64>) {
|
||||
else {
|
||||
return;
|
||||
};
|
||||
retain_published_predecessors_from(session, window, previous_first_proxy_seq);
|
||||
}
|
||||
|
||||
pub(crate) fn retain_published_predecessors_from(
|
||||
session: &HlsSession,
|
||||
window: &mut Vec<u64>,
|
||||
previous_first_proxy_seq: u64,
|
||||
) {
|
||||
let Some(&first_proxy_seq) = window.first() else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -84,6 +84,8 @@ pub struct HlsCacheResponseContext {
|
||||
pub segment_repair: Arc<HlsSegmentRepairManager>,
|
||||
pub qos_meter: Arc<ArcSwapOption<StreamMeterHandle>>,
|
||||
pub media_activity_marker: Option<HlsMediaActivityMarker>,
|
||||
/// Only GET may reserve a lease's progressive opportunity; HEAD waits for metadata.
|
||||
pub request_method: axum::http::Method,
|
||||
pub now_ms: u64,
|
||||
}
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ impl HlsCacheResponseContext {
|
||||
segment_repair,
|
||||
qos_meter: Arc::new(ArcSwapOption::from(qos_meter)),
|
||||
media_activity_marker,
|
||||
request_method: axum::http::Method::GET,
|
||||
now_ms,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,24 +71,16 @@ async fn prepare_hls_revision_response(
|
||||
(Duration::from_secs(startup.config.max_progressive_reader_lifetime_secs.get()), startup.first_data_timeout_ms)
|
||||
};
|
||||
let deadline = deadline.min(remaining_wait_deadline(context.now_ms, wait_ms));
|
||||
let (revision, progressive) = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
let Some(revision) = leases.published_segment_revision(&context.hls_access_lease_id, segment_file.proxy_seq)
|
||||
else {
|
||||
return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing);
|
||||
};
|
||||
let full_request = range.is_none_or(|range| range.as_bytes() == b"bytes=0-");
|
||||
let progressive = revision.revision().key.kind == super::super::SegmentRevisionKind::Raw
|
||||
&& full_request
|
||||
&& leases.try_claim_progressive_startup(
|
||||
&context.hls_access_lease_id,
|
||||
segment_file.proxy_seq,
|
||||
context.now_ms,
|
||||
);
|
||||
(revision, progressive)
|
||||
};
|
||||
let metadata = match revision_metadata(revision.revision(), progressive, deadline).await {
|
||||
Ok(metadata) => metadata,
|
||||
let (revision, metadata) = match resolve_revision_metadata(
|
||||
marker,
|
||||
segment_file,
|
||||
range,
|
||||
context.request_method == axum::http::Method::GET,
|
||||
deadline,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resolved) => resolved,
|
||||
Err(failure) => return HlsResourceServeOutcome::Failure(failure),
|
||||
};
|
||||
let authorized_at_ms = current_time_millis();
|
||||
@@ -154,6 +146,95 @@ fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::I
|
||||
tokio::time::Instant::now() + Duration::from_millis(wait_ms.saturating_sub(elapsed_ms))
|
||||
}
|
||||
|
||||
async fn resolve_revision_metadata(
|
||||
marker: &super::HlsMediaActivityMarker,
|
||||
segment_file: &HlsSegmentFile,
|
||||
range: Option<&HeaderValue>,
|
||||
is_get: bool,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<(crate::SegmentRevisionGuard, Option<CachedSegmentMetadata>), HlsResourceServeFailure> {
|
||||
use crate::lease::HlsRevisionRecoveryOutcome;
|
||||
// Reserve before waiting for recovery or the first body bytes. A concurrent
|
||||
// request must not steal the opportunity from a headers-only first request.
|
||||
let startup_claimed = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
let now_ms = current_time_millis();
|
||||
if !leases.media_identity_is_current(&marker.lease_id, &marker.proxy_session_id, marker.lease_identity, now_ms)
|
||||
{
|
||||
return Err(HlsResourceServeFailure::Expired);
|
||||
}
|
||||
if leases.published_segment_revision(&marker.lease_id, segment_file.proxy_seq).is_none() {
|
||||
return Err(HlsResourceServeFailure::Missing);
|
||||
}
|
||||
let full_request = is_get && range.is_none_or(|range| range.as_bytes() == b"bytes=0-");
|
||||
full_request && leases.try_claim_progressive_startup(&marker.lease_id, segment_file.proxy_seq, now_ms)
|
||||
};
|
||||
let mut last_outcome = None;
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 });
|
||||
}
|
||||
let now_ms = current_time_millis();
|
||||
let outcome = marker
|
||||
.manager
|
||||
.recover_access_lease_revision(
|
||||
&marker.lease_id,
|
||||
&marker.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
segment_file.proxy_seq,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
if outcome == HlsRevisionRecoveryOutcome::ReentryRequired {
|
||||
return Err(HlsResourceServeFailure::Missing);
|
||||
}
|
||||
let (revision, progressive) = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
if !leases.media_identity_is_current(
|
||||
&marker.lease_id,
|
||||
&marker.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
now_ms,
|
||||
) {
|
||||
return Err(HlsResourceServeFailure::Expired);
|
||||
}
|
||||
let revision = leases
|
||||
.published_segment_revision(&marker.lease_id, segment_file.proxy_seq)
|
||||
.ok_or(HlsResourceServeFailure::Missing)?;
|
||||
if outcome == HlsRevisionRecoveryOutcome::RetryPending {
|
||||
if last_outcome != Some(outcome) {
|
||||
let session = marker.session.read().await;
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=retry_pending",
|
||||
crate::safe_hls_access_lease_id(&marker.lease_id), segment_file.proxy_seq,
|
||||
session.activity.origin_work_generation, revision.revision().id(),
|
||||
revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire),
|
||||
revision.revision().was_exposed(),
|
||||
);
|
||||
}
|
||||
last_outcome = Some(outcome);
|
||||
drop(leases);
|
||||
tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await;
|
||||
continue;
|
||||
}
|
||||
let progressive = startup_claimed && revision.revision().key.kind == crate::SegmentRevisionKind::Raw;
|
||||
(revision, progressive)
|
||||
};
|
||||
match revision_metadata(revision.revision(), progressive, deadline).await {
|
||||
Ok(metadata) => return Ok((revision, metadata)),
|
||||
Err(HlsResourceServeFailure::PermanentFailed { .. }) => {
|
||||
// A pending revision may fail while this request waits for EOF.
|
||||
// Re-resolve through the same fenced transaction and deadline.
|
||||
if !matches!(revision.revision().state(), SegmentRevisionState::Failed) {
|
||||
return Err(HlsResourceServeFailure::PermanentFailed { status: None });
|
||||
}
|
||||
tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await;
|
||||
}
|
||||
Err(failure) => return Err(failure),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A progressive claim streams a publishable pending revision; every other
|
||||
/// request waits for the complete revision file.
|
||||
async fn revision_metadata(
|
||||
@@ -162,11 +243,26 @@ async fn revision_metadata(
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<CachedSegmentMetadata>, HlsResourceServeFailure> {
|
||||
if progressive {
|
||||
return match revision.state() {
|
||||
SegmentRevisionState::Complete(metadata) => Ok(Some(metadata)),
|
||||
SegmentRevisionState::Pending if revision.is_publishable() => Ok(None),
|
||||
_ => Err(HlsResourceServeFailure::PermanentFailed { status: None }),
|
||||
};
|
||||
let mut changes = revision.subscribe();
|
||||
loop {
|
||||
let state = changes.borrow_and_update().clone();
|
||||
match state {
|
||||
SegmentRevisionState::Complete(metadata) => return Ok(Some(metadata)),
|
||||
SegmentRevisionState::Failed => {
|
||||
return Err(HlsResourceServeFailure::PermanentFailed { status: None });
|
||||
}
|
||||
SegmentRevisionState::Pending if revision.can_expose() => return Ok(None),
|
||||
SegmentRevisionState::Pending => {}
|
||||
}
|
||||
// Optimistic successors can be listed before any body byte arrives.
|
||||
// Without replay (or after replay exhaustion) wait for the complete
|
||||
// immutable file instead; never expose a missing prefix as a 404.
|
||||
match tokio::time::timeout_at(deadline, changes.changed()).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) => return Err(HlsResourceServeFailure::PermanentFailed { status: None }),
|
||||
Err(_) => return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }),
|
||||
}
|
||||
}
|
||||
}
|
||||
match revision.wait_complete(deadline).await {
|
||||
Ok(metadata) => Ok(Some(metadata)),
|
||||
|
||||
@@ -67,6 +67,7 @@ async fn live_media_marker(
|
||||
struct CachedSegmentFixture {
|
||||
_temp_dir: tempfile::TempDir,
|
||||
manager: Arc<HlsProxyManager>,
|
||||
active_users: Arc<tuliprox_session::ActiveUserManager>,
|
||||
segment_cache: Arc<HlsSegmentCache>,
|
||||
session: HlsSessionHandle,
|
||||
proxy_session_id: ProxySessionId,
|
||||
@@ -80,8 +81,15 @@ struct CachedSegmentFixture {
|
||||
|
||||
impl CachedSegmentFixture {
|
||||
async fn new(bytes: Vec<u8>) -> Self {
|
||||
let fixture = Self::without_lease_user_accounting(bytes).await;
|
||||
fixture.manager.bind_lease_user_accounting(&fixture.active_users);
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn without_lease_user_accounting(bytes: Vec<u8>) -> Self {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300));
|
||||
let active_users = crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users;
|
||||
let segment_cache = Arc::clone(manager.segment_cache());
|
||||
let session = manager.get_or_create_session(HlsSessionKey::new(1, "range-stream"), b"secret", 1_000).await;
|
||||
let proxy_session_id = session.read().await.proxy_session_id.clone();
|
||||
@@ -137,6 +145,7 @@ impl CachedSegmentFixture {
|
||||
Self {
|
||||
_temp_dir: temp_dir,
|
||||
manager,
|
||||
active_users,
|
||||
segment_cache,
|
||||
session,
|
||||
proxy_session_id,
|
||||
@@ -202,6 +211,7 @@ async fn publish_revision_fixture(
|
||||
fixture.segment_file.proxy_seq,
|
||||
if raw { crate::SegmentRevisionKind::Raw } else { crate::SegmentRevisionKind::Processed },
|
||||
)?;
|
||||
revision.revision().bind_origin(&*fixture.session.read().await);
|
||||
if raw {
|
||||
let mut replay =
|
||||
crate::progressive_startup::ProgressiveReplay::new(&budget).ok_or_else(|| std::io::Error::other("slot"))?;
|
||||
@@ -243,6 +253,7 @@ async fn publish_revision_fixture(
|
||||
});
|
||||
let snapshot = crate::HlsLeaseManifestSnapshot {
|
||||
startup_revisions: Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: config.mode,
|
||||
retained_start_seq: fixture.segment_file.proxy_seq,
|
||||
revisions,
|
||||
@@ -283,8 +294,10 @@ async fn publish_revision_fixture(
|
||||
}
|
||||
|
||||
mod admission;
|
||||
mod first_requested;
|
||||
mod http;
|
||||
mod lifecycle;
|
||||
mod policy;
|
||||
mod recovery;
|
||||
mod revision_recovery;
|
||||
mod terminal;
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//! Startup follows the player's first full GET, including a headers-only successor.
|
||||
use super::*;
|
||||
use crate::sync_ext::MutexExt;
|
||||
|
||||
const BODY: &[u8] = b"prefix-tail";
|
||||
|
||||
struct FirstRequestFixture {
|
||||
cached: CachedSegmentFixture,
|
||||
revisions: std::collections::BTreeMap<u64, crate::SegmentRevisionGuard>,
|
||||
}
|
||||
|
||||
impl FirstRequestFixture {
|
||||
async fn new(mode: shared::model::HlsStartupMode, with_prefix: bool, with_replay: bool) -> std::io::Result<Self> {
|
||||
let cached = CachedSegmentFixture::new(BODY.to_vec()).await;
|
||||
let head = publish_revision_fixture(&cached, true).await?;
|
||||
let mut revisions = std::collections::BTreeMap::from([(12, head)]);
|
||||
{
|
||||
let mut session = cached.session.write().await;
|
||||
session.startup.as_mut().expect("startup").config.mode = mode;
|
||||
let store = Arc::clone(&session.startup.as_ref().expect("startup").store);
|
||||
let budget = Arc::clone(&session.startup.as_ref().expect("startup").budget);
|
||||
for seq in [13, 14] {
|
||||
let mut entry = session.segments[&12].clone();
|
||||
entry.proxy_seq = seq;
|
||||
entry.origin_key.host_local_sequence = seq;
|
||||
entry.origin_key.host_local_index = u32::try_from(seq).expect("fixture sequence");
|
||||
entry.cache_key = SegmentCacheKey::new(cached.proxy_session_id.clone(), seq, "ts");
|
||||
entry.media_start_ms = seq * 4000;
|
||||
entry.status = SegmentCacheStatus::Fetching {
|
||||
priority: crate::SegmentFetchPriority::Demand,
|
||||
started_at_ms: current_time_millis(),
|
||||
};
|
||||
session.segments.insert(seq, entry);
|
||||
let revision = store.create(cached.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?;
|
||||
revision.revision().bind_origin(&session);
|
||||
if with_replay {
|
||||
let mut replay = crate::progressive_startup::ProgressiveReplay::new(&budget)
|
||||
.ok_or_else(|| std::io::Error::other("replay slot"))?;
|
||||
if with_prefix {
|
||||
replay.push(b"prefix")?;
|
||||
revision.revision().prefix_available.store(6, Ordering::Release);
|
||||
}
|
||||
*revision.revision().replay.lock_unpoisoned() = Some(replay);
|
||||
}
|
||||
session.startup.as_mut().expect("startup").revisions.insert(seq, revision.clone());
|
||||
revisions.insert(seq, revision);
|
||||
}
|
||||
}
|
||||
let mut leases = cached.manager.access_leases().write().await;
|
||||
let mut snapshot = leases
|
||||
.response_snapshot(&cached.lease_id, &cached.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
let mut segments = snapshot.visible_segments.to_vec();
|
||||
for seq in [13, 14] {
|
||||
let mut segment = segments[0].clone();
|
||||
segment.proxy_seq = seq;
|
||||
segment.uri = format!("{seq}.ts").into();
|
||||
segments.push(segment);
|
||||
}
|
||||
snapshot.visible_segments = Arc::from(segments);
|
||||
snapshot.last_proxy_seq = 14;
|
||||
snapshot.playlist_duration_ms = 12_000;
|
||||
snapshot.last_visible_media_end_ms = 12_000;
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode,
|
||||
retained_start_seq: 12,
|
||||
revisions: revisions.clone(),
|
||||
}));
|
||||
let now = current_time_millis();
|
||||
let guard =
|
||||
leases.prepare_manifest_publication(&cached.lease_id, &cached.proxy_session_id, now).expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(&cached.lease_id, &cached.proxy_session_id, guard, snapshot, now)
|
||||
.is_committed());
|
||||
drop(leases);
|
||||
Ok(Self { cached, revisions })
|
||||
}
|
||||
|
||||
async fn serve(&self, seq: u64, range: Option<&str>) -> HlsResourceServeOutcome {
|
||||
self.serve_method(seq, range, axum::http::Method::GET).await
|
||||
}
|
||||
|
||||
async fn serve_method(&self, seq: u64, range: Option<&str>, method: axum::http::Method) -> HlsResourceServeOutcome {
|
||||
let context = HlsCacheResponseContext {
|
||||
hls_access_lease_id: self.cached.lease_id.clone(),
|
||||
log_identity: self.cached.context.log_identity.clone(),
|
||||
cache_duration_seconds: self.cached.context.cache_duration_seconds,
|
||||
metrics: Arc::clone(&self.cached.context.metrics),
|
||||
segment_repair: Arc::clone(&self.cached.context.segment_repair),
|
||||
qos_meter: Arc::clone(&self.cached.context.qos_meter),
|
||||
media_activity_marker: self.cached.context.media_activity_marker.clone(),
|
||||
request_method: method,
|
||||
now_ms: current_time_millis(),
|
||||
};
|
||||
serve_hls_segment_cache_outcome(
|
||||
Arc::clone(&self.cached.segment_cache),
|
||||
Arc::clone(&self.cached.session),
|
||||
HlsSegmentFile { proxy_seq: seq, extension: "ts".into() },
|
||||
range.map(header),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn prefix(&self, seq: u64) -> std::io::Result<()> {
|
||||
let revision = self.revisions[&seq].revision();
|
||||
revision.replay.lock_unpoisoned().as_mut().expect("replay").push(b"prefix")?;
|
||||
revision.prefix_available.store(6, Ordering::Release);
|
||||
revision.prefix_changed();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete(&self, seq: u64) -> std::io::Result<()> {
|
||||
let cache_key = self.cached.session.read().await.segments[&seq].cache_key.clone();
|
||||
let metadata = self.cached.segment_cache.write_bytes_and_commit(&cache_key, BODY).await?;
|
||||
assert!(self.revisions[&seq].revision().complete(metadata));
|
||||
self.revisions[&seq].revision().replay.lock_unpoisoned().take();
|
||||
self.cached.session.write().await.segments.get_mut(&seq).expect("entry").status = SegmentCacheStatus::Ready {
|
||||
content_length: u64::try_from(BODY.len()).expect("body length"),
|
||||
ready_at_ms: current_time_millis(),
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_successor_streams_before_eof_in_both_raw_startup_modes() -> std::io::Result<()> {
|
||||
for mode in [shared::model::HlsStartupMode::Progressive, shared::model::HlsStartupMode::Optimistic] {
|
||||
let fixture = FirstRequestFixture::new(mode, true, true).await?;
|
||||
let response = ready_response(fixture.serve(13, None).await);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(
|
||||
body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(),
|
||||
Some(b"prefix".as_slice())
|
||||
);
|
||||
assert!(fixture.revisions[&13].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&12].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&13].revision().is_complete());
|
||||
let subsequent = fixture.serve(14, None);
|
||||
tokio::pin!(subsequent);
|
||||
assert!(futures::poll!(subsequent.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
let subsequent = ready_response(subsequent.await);
|
||||
assert!(subsequent.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(subsequent.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
fixture.complete(13).await?;
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"-tail".as_slice()));
|
||||
assert!(body.next().await.is_none());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn headers_only_first_get_keeps_the_claim_until_its_own_prefix_arrives() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
let first = fixture.serve(13, Some("bytes=0-"));
|
||||
let later = fixture.serve(14, None);
|
||||
tokio::pin!(first, later);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(later.as_mut()).is_pending());
|
||||
fixture.prefix(14)?;
|
||||
assert!(futures::poll!(later.as_mut()).is_pending(), "a later GET still waits for EOF");
|
||||
fixture.prefix(13)?;
|
||||
let response = ready_response(first.await);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
assert!(futures::poll!(later.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
assert!(ready_response(later.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn headers_only_first_get_times_out_temporarily_and_preserves_the_lease() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
fixture.cached.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25;
|
||||
let failure = match fixture.serve(13, None).await {
|
||||
HlsResourceServeOutcome::Failure(failure) => failure,
|
||||
HlsResourceServeOutcome::Ready(_) => panic!("the origin has not supplied a prefix"),
|
||||
};
|
||||
let response = crate::response::cache::hls_resource_failure_default_response(failure);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response.headers().contains_key(header::RETRY_AFTER));
|
||||
assert!(fixture
|
||||
.cached
|
||||
.manager
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.cached.lease_id,
|
||||
&fixture.cached.proxy_session_id,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_some());
|
||||
fixture.complete(13).await?;
|
||||
let retry = ready_response(fixture.serve(13, None).await);
|
||||
assert!(retry.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(retry.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_metadata_probe_does_not_consume_the_first_get_opportunity() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
let head = fixture.serve_method(12, None, axum::http::Method::HEAD);
|
||||
tokio::pin!(head);
|
||||
assert!(futures::poll!(head.as_mut()).is_pending(), "HEAD waits for complete metadata");
|
||||
let first_get = ready_response(fixture.serve(13, None).await);
|
||||
assert!(!first_get.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert!(fixture.revisions[&13].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&12].revision().was_exposed());
|
||||
fixture.complete(12).await?;
|
||||
assert!(ready_response(head.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_get_without_replay_or_after_unexposed_replay_exhaustion_waits_for_complete_bytes() -> std::io::Result<()>
|
||||
{
|
||||
for replay in [false, true] {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, replay).await?;
|
||||
let first = fixture.serve(13, None);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
if replay {
|
||||
fixture.revisions[&13].revision().replay_exhausted();
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
}
|
||||
fixture.complete(13).await?;
|
||||
let response = ready_response(first.await);
|
||||
assert!(response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(response.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
assert!(!fixture.revisions[&13].revision().was_exposed());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_first_get_consumes_startup_but_a_partial_range_does_not() -> std::io::Result<()> {
|
||||
for partial in [false, true] {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
fixture.complete(12).await?;
|
||||
let first = ready_response(fixture.serve(12, if partial { Some("bytes=2-5") } else { None }).await);
|
||||
assert_eq!(first.status(), if partial { StatusCode::PARTIAL_CONTENT } else { StatusCode::OK });
|
||||
first.into_body().collect().await.map_err(std::io::Error::other)?;
|
||||
let next = fixture.serve(13, None);
|
||||
tokio::pin!(next);
|
||||
if partial {
|
||||
let response = ready_response(next.await);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
} else {
|
||||
assert!(futures::poll!(next.as_mut()).is_pending());
|
||||
fixture.complete(13).await?;
|
||||
assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_uri_does_not_consume_startup_and_cancelled_first_get_does() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
assert!(matches!(
|
||||
fixture.serve(99, None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing)
|
||||
));
|
||||
{
|
||||
let first = fixture.serve(13, None);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
}
|
||||
fixture.prefix(14)?;
|
||||
let next = fixture.serve(14, None);
|
||||
tokio::pin!(next);
|
||||
assert!(futures::poll!(next.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_successor_body_failure_is_terminal_for_the_exposed_revision() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
let response = ready_response(fixture.serve(13, None).await);
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
fixture.revisions[&13].revision().fail();
|
||||
assert!(body.next().await.is_some_and(|chunk| chunk.is_err()));
|
||||
assert!(body.next().await.is_none());
|
||||
assert!(matches!(
|
||||
fixture.serve(13, None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing)
|
||||
));
|
||||
assert!(fixture
|
||||
.cached
|
||||
.manager
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.cached.lease_id,
|
||||
&fixture.cached.proxy_session_id,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
@@ -131,7 +131,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segment_failure() -> std::io::Result<()> {
|
||||
async fn optimistic_successor_without_a_retry_ends_only_its_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await;
|
||||
let seq = fixture.segment_file.proxy_seq;
|
||||
let head_seq = seq.checked_sub(1).ok_or_else(|| std::io::Error::other("fixture head"))?;
|
||||
@@ -164,6 +164,7 @@ async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segmen
|
||||
};
|
||||
let snapshot = crate::HlsLeaseManifestSnapshot {
|
||||
startup_revisions: Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: config.mode,
|
||||
retained_start_seq: head_seq,
|
||||
revisions,
|
||||
@@ -195,12 +196,17 @@ async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segmen
|
||||
.commit_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, guard, snapshot, now)
|
||||
.is_committed());
|
||||
}
|
||||
// Listing does not expose a revision; only streaming bytes does. The origin
|
||||
// failing a listed successor therefore fails just that one segment.
|
||||
// Listing does not expose bytes. With no usable retry and a terminal fetch
|
||||
// state, the controlled failure transition retires this lease.
|
||||
successor.revision().fail();
|
||||
assert!(matches!(
|
||||
fixture.serve(None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::PermanentFailed { .. })
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing | HlsResourceServeFailure::Expired)
|
||||
));
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,978 @@
|
||||
use super::*;
|
||||
use crate::lease::HlsRevisionRecoveryOutcome;
|
||||
|
||||
async fn permanent_failure_with_counted_user(fixture: &CachedSegmentFixture) -> std::io::Result<()> {
|
||||
failed_optimistic_revision(fixture, 6).await?;
|
||||
assert!(fixture
|
||||
.manager
|
||||
.activate_access_lease(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
current_time_millis(),
|
||||
crate::HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 },
|
||||
)
|
||||
.await
|
||||
.is_activated());
|
||||
fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None };
|
||||
let lease = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease");
|
||||
let mut user = tuliprox_core::model::ProxyUserCredentials::default();
|
||||
user.username = lease.username.clone();
|
||||
user.max_connections = 1;
|
||||
fixture
|
||||
.active_users
|
||||
.create_user_session(tuliprox_session::CreateUserSessionParams {
|
||||
user: &user,
|
||||
session_token: &lease.user_session_token,
|
||||
virtual_id: 1,
|
||||
provider: "synthetic-provider",
|
||||
stream_url: "http://synthetic.invalid/live.m3u8",
|
||||
addr: &"127.0.0.1:12345".parse().expect("address"),
|
||||
connection_permission: shared::model::UserConnectionPermission::Allowed,
|
||||
connection_kind: Some(tuliprox_session::ConnectionKind::Normal),
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
fixture.active_users.mark_grace_active(&lease.username, &lease.user_session_token).await;
|
||||
let version =
|
||||
fixture.active_users.grace_active_version(&lease.username, &lease.user_session_token).await.expect("version");
|
||||
fixture.active_users.activate_grace_active(&lease.username, &lease.user_session_token, version).await;
|
||||
assert_eq!(fixture.active_users.user_connections(&lease.username).await, 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_releases_the_counted_user_reservation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
assert_eq!(
|
||||
fixture.active_users.connection_permission("test-user", 1, 0).await,
|
||||
shared::model::UserConnectionPermission::Exhausted
|
||||
);
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert_eq!(
|
||||
fixture.active_users.connection_permission("test-user", 1, 0).await,
|
||||
shared::model::UserConnectionPermission::Allowed
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_preserves_another_counted_session_of_the_same_user() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let mut user = tuliprox_core::model::ProxyUserCredentials::default();
|
||||
user.username = "test-user".into();
|
||||
user.max_connections = 2;
|
||||
let token = "other-playback";
|
||||
fixture
|
||||
.active_users
|
||||
.create_user_session(tuliprox_session::CreateUserSessionParams {
|
||||
user: &user,
|
||||
session_token: token,
|
||||
virtual_id: 2,
|
||||
provider: "synthetic-provider",
|
||||
stream_url: "http://synthetic.invalid/other.m3u8",
|
||||
addr: &"127.0.0.1:12346".parse().expect("address"),
|
||||
connection_permission: shared::model::UserConnectionPermission::Allowed,
|
||||
connection_kind: Some(tuliprox_session::ConnectionKind::Normal),
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
fixture.active_users.mark_grace_active(&user.username, token).await;
|
||||
let version = fixture.active_users.grace_active_version(&user.username, token).await.expect("version");
|
||||
fixture.active_users.activate_grace_active(&user.username, token, version).await;
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 2);
|
||||
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 1);
|
||||
assert!(fixture.active_users.release_session_streams_and_counted_reservation(&user.username, token).await);
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn normal_lifecycle_completion_uses_the_same_counted_user_release() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let expires = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease")
|
||||
.valid_until_ms
|
||||
+ 1;
|
||||
let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default());
|
||||
fixture
|
||||
.manager
|
||||
.handle_lifecycle_event(
|
||||
&fixture.active_users,
|
||||
&ctx.active_provider,
|
||||
crate::HlsLifecycleEvent {
|
||||
key: crate::HlsLifecycleEventKey::AccessLeaseValidity {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
due_at_ms: expires,
|
||||
},
|
||||
expires,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expires,)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_survives_request_cancellation_during_cleanup() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
let reader = fixture.session.read().await;
|
||||
{
|
||||
let request = recovery(&fixture);
|
||||
tokio::pin!(request);
|
||||
assert!(futures::poll!(request.as_mut()).is_pending());
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.access_leases().try_read().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup holds its lease fence while waiting for the session writer");
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
// The request deadline drops this future while cleanup is blocked.
|
||||
}
|
||||
drop(reader);
|
||||
let settled = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if fixture.manager.qos().is_empty().await
|
||||
&& fixture.manager.segment_repair().stats().await.windows == 0
|
||||
&& fixture.manager.pending_lease_completions_for_test() == 0
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(settled.is_ok(), "cancelled cleanup left QoS or repair state");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_reentry_waiters_share_completion_when_one_is_cancelled() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
let reader = fixture.session.read().await;
|
||||
let second = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(second);
|
||||
{
|
||||
let first = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
}
|
||||
drop(reader);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), second).await.expect("owned completion"));
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 0);
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_reentry_job_preserves_new_incarnation_and_its_user_reservation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let mut replacement =
|
||||
leases.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()).expect("lease");
|
||||
replacement.issued_at_ms += 1;
|
||||
let replacement_issued_at = replacement.issued_at_ms;
|
||||
assert!(leases.prepare_access_lease(replacement));
|
||||
drop(leases);
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 1);
|
||||
assert_eq!(fixture.manager.qos().len().await, 1);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),)
|
||||
.await
|
||||
.expect("new incarnation")
|
||||
.issued_at_ms,
|
||||
replacement_issued_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_reentry_waiting_for_accounting_preserves_replacement_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::without_lease_user_accounting(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
{
|
||||
let request = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(request);
|
||||
assert!(futures::poll!(request.as_mut()).is_pending());
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
}
|
||||
// The owned job outlives its caller while waiting for runtime attachment.
|
||||
// It must revalidate the lease after attachment, before releasing anything.
|
||||
let replacement_issued_at = {
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let mut replacement = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease");
|
||||
replacement.issued_at_ms += 1;
|
||||
let issued_at = replacement.issued_at_ms;
|
||||
assert!(leases.prepare_access_lease(replacement));
|
||||
issued_at
|
||||
};
|
||||
fixture.manager.bind_lease_user_accounting(&fixture.active_users);
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.pending_lease_completions_for_test() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("stale completion settles after runtime attachment");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 1);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("replacement lease")
|
||||
.issued_at_ms,
|
||||
replacement_issued_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_lifecycle_and_reentry_cleanup_survive_cancellation_of_all_waiters() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
let (expires_at, guard) = {
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let expires_at = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.valid_until_ms
|
||||
+ 1;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
(expires_at, guard)
|
||||
};
|
||||
let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default());
|
||||
let session_reader = fixture.session.read().await;
|
||||
{
|
||||
let lifecycle = fixture.manager.handle_lifecycle_event(
|
||||
&fixture.active_users,
|
||||
&ctx.active_provider,
|
||||
crate::HlsLifecycleEvent {
|
||||
key: crate::HlsLifecycleEventKey::AccessLeaseValidity {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
due_at_ms: expires_at,
|
||||
},
|
||||
expires_at,
|
||||
);
|
||||
let reentry = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(lifecycle, reentry);
|
||||
assert!(futures::poll!(lifecycle.as_mut()).is_pending());
|
||||
assert!(futures::poll!(reentry.as_mut()).is_pending());
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.access_leases().try_read().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup waits for the session writer while holding its lease fence");
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 2);
|
||||
// Both caller futures are cancelled while their owned jobs are blocked.
|
||||
}
|
||||
drop(session_reader);
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.pending_lease_completions_for_test() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned cleanup settles after all waiters are cancelled");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture.manager.qos().is_empty().await);
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn failed_optimistic_revision(
|
||||
fixture: &CachedSegmentFixture,
|
||||
prefix: u64,
|
||||
) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
let revision = publish_revision_fixture(fixture, true).await?;
|
||||
let mut session = fixture.session.write().await;
|
||||
session.startup.as_mut().expect("startup").config.mode = shared::model::HlsStartupMode::Optimistic;
|
||||
drop(session);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let mut snapshot = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).mode =
|
||||
shared::model::HlsStartupMode::Optimistic;
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.is_committed());
|
||||
revision.revision().prefix_available.store(prefix, Ordering::Release);
|
||||
revision.revision().fail();
|
||||
drop(leases);
|
||||
let mut session = fixture.session.write().await;
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = SegmentCacheStatus::Fetching {
|
||||
priority: crate::SegmentFetchPriority::Demand,
|
||||
started_at_ms: current_time_millis(),
|
||||
};
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
async fn install_completed_retry(fixture: &CachedSegmentFixture) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
let mut session = fixture.session.write().await;
|
||||
let startup = session.startup.as_ref().expect("startup");
|
||||
let retry = startup.store.create(
|
||||
fixture.proxy_session_id.clone(),
|
||||
fixture.segment_file.proxy_seq,
|
||||
crate::SegmentRevisionKind::Raw,
|
||||
)?;
|
||||
retry.revision().bind_origin(&session);
|
||||
let entry = session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry");
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() };
|
||||
assert!(retry.revision().complete(crate::CachedSegmentMetadata {
|
||||
path: fixture.segment_cache.object_path(&entry.cache_key),
|
||||
size: fixture.full_size,
|
||||
}));
|
||||
session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone());
|
||||
Ok(retry)
|
||||
}
|
||||
|
||||
async fn recovery(fixture: &CachedSegmentFixture) -> HlsRevisionRecoveryOutcome {
|
||||
let marker = fixture.context.media_activity_marker.as_ref().expect("marker");
|
||||
fixture
|
||||
.manager
|
||||
.recover_access_lease_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn install_spooled_retry(fixture: &CachedSegmentFixture) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (store, cache_key) = {
|
||||
let session = fixture.session.read().await;
|
||||
(
|
||||
session.startup.as_ref().expect("startup").store.clone(),
|
||||
session.segments[&fixture.segment_file.proxy_seq].cache_key.clone(),
|
||||
)
|
||||
};
|
||||
let retry = store.create(
|
||||
fixture.proxy_session_id.clone(),
|
||||
fixture.segment_file.proxy_seq,
|
||||
crate::SegmentRevisionKind::Raw,
|
||||
)?;
|
||||
let reservation = fixture.segment_cache.reserve_revision_peak(&cache_key, fixture.full_size * 5).await?;
|
||||
let (mut file, pin) =
|
||||
fixture.segment_cache.create_revision_spool_with_budget(&retry.revision().key, Some(&reservation)).await?;
|
||||
let bytes = tokio::fs::read(fixture.segment_cache.object_path(&cache_key)).await?;
|
||||
file.write_all(&bytes).await?;
|
||||
file.flush().await?;
|
||||
drop(file);
|
||||
let path = fixture.segment_cache.object_path(&retry.revision().key);
|
||||
fixture.segment_cache.account_reserved_spool(&path, fixture.full_size, &reservation)?;
|
||||
*retry.revision().file_pin.lock().expect("pin") = Some(pin);
|
||||
assert!(retry.revision().complete(crate::CachedSegmentMetadata { path, size: fixture.full_size }));
|
||||
let mut session = fixture.session.write().await;
|
||||
retry.revision().bind_origin(&session);
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() };
|
||||
session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone());
|
||||
Ok(retry)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_revision_retry_recovers_same_get_and_snapshot_without_playlist_refresh() -> std::io::Result<()> {
|
||||
for prefix in [0, 6, 3_230_293] {
|
||||
let fixture = CachedSegmentFixture::new(b"complete-retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, prefix).await?;
|
||||
let before = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease before");
|
||||
let availability =
|
||||
fixture.manager.access_leases().read().await.availability_evidence_generation(&fixture.proxy_session_id);
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
let response = ready_response(fixture.serve(None).await);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"complete-retry-body".as_slice());
|
||||
let after = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("same lease");
|
||||
let snapshot = after.last_manifest_snapshot.expect("snapshot");
|
||||
let previous_snapshot = before.last_manifest_snapshot.expect("previous snapshot");
|
||||
assert!(snapshot.snapshot_generation > previous_snapshot.snapshot_generation);
|
||||
assert_eq!(snapshot.visible_segments, previous_snapshot.visible_segments);
|
||||
assert_eq!(snapshot.first_proxy_seq, previous_snapshot.first_proxy_seq);
|
||||
assert_eq!(
|
||||
snapshot.startup_revisions.as_ref().expect("revisions").revisions[&fixture.segment_file.proxy_seq],
|
||||
retry
|
||||
);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert_ne!(leases.availability_evidence_generation(&fixture.proxy_session_id), availability);
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(retry));
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.is_committed());
|
||||
assert!(!failed.revision().was_exposed());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_pending_is_503_without_ending_the_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
fixture.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25;
|
||||
let failure = match fixture.serve(None).await {
|
||||
HlsResourceServeOutcome::Failure(failure) => failure,
|
||||
HlsResourceServeOutcome::Ready(_) => panic!("retry must wait"),
|
||||
};
|
||||
let response = crate::response::cache::hls_resource_failure_default_response(failure);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response.headers().contains_key(header::RETRY_AFTER));
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_some());
|
||||
install_completed_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn waiting_get_resolves_a_retry_that_finishes_before_its_deadline() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
let request = fixture.serve(None);
|
||||
let complete = async {
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
install_completed_retry(&fixture).await
|
||||
};
|
||||
let (response, retry) = tokio::join!(request, complete);
|
||||
retry?;
|
||||
let response = ready_response(response);
|
||||
assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"retry-body".as_slice());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exposed_failure_ends_only_the_old_lease_and_cannot_append_retry_bytes() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let old = publish_revision_fixture(&fixture, true).await?;
|
||||
let base_snapshot = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("old lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("old snapshot");
|
||||
let response = ready_response(fixture.serve(None).await);
|
||||
let mut stream = response.into_body().into_data_stream();
|
||||
assert_eq!(stream.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
old.revision().fail();
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
assert!(stream.next().await.is_some_and(|chunk| chunk.is_err()));
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::Unchanged);
|
||||
assert!(fixture.manager.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.is_some());
|
||||
assert_eq!(
|
||||
fixture.session.read().await.startup.as_ref().expect("shared startup").revisions
|
||||
[&fixture.segment_file.proxy_seq],
|
||||
retry
|
||||
);
|
||||
let new_id = HlsAccessLeaseId("replacement-viewer".into());
|
||||
let marker = live_media_marker(&fixture.manager, &fixture.session, new_id.clone()).await;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert!(leases.media_identity_is_current(
|
||||
&new_id,
|
||||
&fixture.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
current_time_millis()
|
||||
));
|
||||
let mut snapshot = base_snapshot;
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
window_plan: None,
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: fixture.segment_file.proxy_seq,
|
||||
revisions: [(fixture.segment_file.proxy_seq, retry)].into_iter().collect(),
|
||||
}));
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&new_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("new publication");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(&new_id, &fixture.proxy_session_id, guard, snapshot, current_time_millis())
|
||||
.is_committed());
|
||||
drop(leases);
|
||||
let context = HlsCacheResponseContext::new(
|
||||
new_id,
|
||||
test_log_identity(),
|
||||
300,
|
||||
fixture.manager.metrics().clone(),
|
||||
fixture.manager.segment_repair().clone(),
|
||||
None,
|
||||
Some(marker),
|
||||
current_time_millis(),
|
||||
);
|
||||
let response = ready_response(
|
||||
serve_hls_segment_cache_outcome(
|
||||
fixture.segment_cache.clone(),
|
||||
fixture.session.clone(),
|
||||
fixture.segment_file.clone(),
|
||||
None,
|
||||
&context,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.into_body().collect().await.expect("new lease body").to_bytes(), b"retry-body".as_slice());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_fences_generation_snapshot_and_lease_incarnation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
install_completed_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let session = fixture.session.read().await;
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()),
|
||||
HlsRevisionRecoveryOutcome::Unchanged
|
||||
);
|
||||
assert!(leases
|
||||
.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, current_time_millis())
|
||||
.is_none());
|
||||
drop(session);
|
||||
drop(leases);
|
||||
// Replacing the same ID does not let a delayed failure end the new viewer.
|
||||
live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await;
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_recovery_commits_once_and_preserves_progressive_quota() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
{
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert!(leases.try_claim_progressive_startup(
|
||||
&fixture.lease_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis()
|
||||
));
|
||||
}
|
||||
install_completed_retry(&fixture).await?;
|
||||
let (first, second) = tokio::join!(recovery(&fixture), recovery(&fixture));
|
||||
assert!(matches!(
|
||||
(first, second),
|
||||
(HlsRevisionRecoveryOutcome::ReplacementBound, HlsRevisionRecoveryOutcome::Unchanged)
|
||||
| (HlsRevisionRecoveryOutcome::Unchanged, HlsRevisionRecoveryOutcome::ReplacementBound)
|
||||
));
|
||||
assert!(!fixture.manager.access_leases().write().await.try_claim_progressive_startup(
|
||||
&fixture.lease_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis()
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn origin_generation_and_mapping_changes_never_install_retired_retry_bytes() -> std::io::Result<()> {
|
||||
for change_generation in [false, true] {
|
||||
let fixture = CachedSegmentFixture::new(b"retired-retry".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
install_completed_retry(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let mut session = fixture.session.write().await;
|
||||
if change_generation {
|
||||
session.activity.origin_work_generation += 1;
|
||||
} else {
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").origin_key.origin_epoch += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()),
|
||||
HlsRevisionRecoveryOutcome::ReentryRequired
|
||||
);
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
install_completed_retry(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let expires = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.valid_until_ms;
|
||||
let session = fixture.session.read().await;
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, expires + 1),
|
||||
HlsRevisionRecoveryOutcome::Unchanged
|
||||
);
|
||||
assert!(leases.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, expires + 1).is_none());
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_completion_recovers_before_get_and_final_failure_reuses_manager_cleanup() -> std::io::Result<()> {
|
||||
for succeeds in [false, true] {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let generation = fixture.session.read().await.activity.origin_work_generation;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
let other_id = HlsAccessLeaseId("unaffected-viewer".into());
|
||||
let other = live_media_marker(&fixture.manager, &fixture.session, other_id.clone()).await;
|
||||
fixture.manager.segment_repair().start_access_lease_window(other_id.clone()).await;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&other_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
let retry = if succeeds {
|
||||
Some(install_completed_retry(&fixture).await?)
|
||||
} else {
|
||||
fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None };
|
||||
None
|
||||
};
|
||||
let runtime = fixture.manager.segment_worker_pool().revision_recovery.get().expect("manager recovery runtime");
|
||||
runtime
|
||||
.recover_segment_leases_after_fetch(
|
||||
&fixture.proxy_session_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
generation.wrapping_sub(1),
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
Some(failed)
|
||||
);
|
||||
runtime
|
||||
.recover_segment_leases_after_fetch(
|
||||
&fixture.proxy_session_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
generation,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
if succeeds {
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
retry
|
||||
);
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 2);
|
||||
assert_eq!(fixture.manager.qos().len().await, 2);
|
||||
} else {
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 1);
|
||||
assert_eq!(fixture.manager.qos().len().await, 1);
|
||||
}
|
||||
assert!(fixture.manager.access_leases().write().await.media_identity_is_current(
|
||||
&other_id,
|
||||
&fixture.proxy_session_id,
|
||||
other.lease_identity,
|
||||
current_time_millis(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_publication_uses_the_same_recovery_and_never_reinstalls_an_older_attempt() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let old_snapshot = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("publication");
|
||||
assert!(fixture
|
||||
.manager
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
old_snapshot.clone(),
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
Some(retry)
|
||||
);
|
||||
// An old request can still contain the failed guard, but it cannot rewind
|
||||
// the successful transaction to that guard or make it complete again.
|
||||
assert!(!failed.revision().complete(crate::CachedSegmentMetadata { path: "retired.ts".into(), size: 1 }));
|
||||
let guard = fixture
|
||||
.manager
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("publication");
|
||||
assert!(fixture
|
||||
.manager
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
old_snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_recovery_cycles_release_old_publication_pins_and_replay_budget() -> std::io::Result<()> {
|
||||
let mut fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
for _ in 0..128 {
|
||||
let marker = live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await;
|
||||
fixture.context.media_activity_marker = Some(marker);
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let (store, budget) = {
|
||||
let session = fixture.session.read().await;
|
||||
let startup = session.startup.as_ref().expect("startup");
|
||||
(startup.store.clone(), startup.budget.clone())
|
||||
};
|
||||
let retry = install_spooled_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
assert_eq!(failed.revision().pin_count(), 1, "no failed guard remains in the lease or snapshot");
|
||||
assert_eq!(budget.usage(), (0, 0));
|
||||
fixture.manager.access_leases().write().await.remove_access_lease(&fixture.lease_id);
|
||||
fixture.session.write().await.startup.as_mut().expect("startup").revisions.clear();
|
||||
drop(failed);
|
||||
drop(retry);
|
||||
let retired = store.retire_unpinned()?;
|
||||
assert_eq!(retired.len(), 2);
|
||||
for revision in retired {
|
||||
fixture.segment_cache.delete_if_inactive(&revision.key).await?;
|
||||
assert!(!fixture.segment_cache.object_path(&revision.key).exists());
|
||||
}
|
||||
assert!(store.retire_unpinned()?.is_empty());
|
||||
assert_eq!(fixture.segment_cache.revision_disk_bytes(), 0);
|
||||
assert_eq!(fixture.segment_cache.remaining_revision_bytes(&fixture.proxy_session_id), 0);
|
||||
assert!(!fixture
|
||||
.segment_cache
|
||||
.has_active_temp_files_for_session_except::<crate::SegmentRevisionKey>(&fixture.proxy_session_id, &[]));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -87,6 +87,7 @@ pub enum HlsSwitchResourceKind {
|
||||
/// Bounded scheduler for live HLS segment demand fetches and prefetches.
|
||||
pub struct HlsSegmentWorkerPool {
|
||||
runtime: ArcSwap<SegmentWorkerRuntime>,
|
||||
pub(crate) revision_recovery: std::sync::OnceLock<Arc<crate::manager::HlsLeaseRuntime>>,
|
||||
access_leases: Arc<RwLock<HlsAccessLeaseStore>>,
|
||||
metrics: Arc<HlsCacheMetrics>,
|
||||
availability_reevaluations: Option<Arc<super::availability_reevaluation::HlsAvailabilityReevaluationCoordinator>>,
|
||||
|
||||
@@ -12,6 +12,7 @@ pub(super) struct SegmentFetchCompletion {
|
||||
pub(super) notifier: Option<Arc<Notify>>,
|
||||
pub(super) scheduled_retry: Option<ScheduledSegmentRetry>,
|
||||
pub(super) evidence_changed_for: Option<ProxySessionId>,
|
||||
pub(super) recovery_generation: Option<u64>,
|
||||
}
|
||||
|
||||
pub(super) struct SegmentFetchCommit {
|
||||
@@ -113,6 +114,7 @@ impl HlsSegmentWorkerPool {
|
||||
notifier,
|
||||
scheduled_retry,
|
||||
evidence_changed_for,
|
||||
recovery_generation: generation_valid.then_some(session.activity.origin_work_generation),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,7 +159,8 @@ pub(super) fn commit_failed_segment_fetch(
|
||||
error: &SegmentFetchError,
|
||||
finished_at_ms: u64,
|
||||
) {
|
||||
let missing_candidate = session.origin_source.archive_reference.is_none()
|
||||
let missing_candidate = !session.uses_fast_start_live_window()
|
||||
&& session.origin_source.archive_reference.is_none()
|
||||
&& session
|
||||
.publishable_origin_head_proxy_seq
|
||||
.is_some_and(|head| snapshot.proxy_seq >= head && snapshot.proxy_seq <= missing_head_bound(session, head))
|
||||
@@ -206,6 +209,9 @@ fn missing_head_bound(session: &super::super::HlsSession, head: u64) -> u64 {
|
||||
/// new origin window: until a client has seen that head, skipping a missing
|
||||
/// object cannot open a hole in published media.
|
||||
pub(crate) fn recompute_unpublished_live_head(session: &mut super::super::HlsSession) {
|
||||
if session.uses_fast_start_live_window() {
|
||||
return;
|
||||
}
|
||||
if session.origin_source.archive_reference.is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -186,6 +186,7 @@ impl HlsSegmentWorkerPool {
|
||||
) -> Self {
|
||||
Self {
|
||||
runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)),
|
||||
revision_recovery: std::sync::OnceLock::new(),
|
||||
access_leases,
|
||||
metrics,
|
||||
availability_reevaluations,
|
||||
@@ -326,11 +327,19 @@ impl HlsSegmentWorkerPool {
|
||||
if mode.publishes_pending_successors() {
|
||||
return false;
|
||||
}
|
||||
let Some(head) = session.publishable_origin_head_proxy_seq else {
|
||||
return true;
|
||||
let heads = if session.uses_fast_start_live_window() {
|
||||
session.live_window_heads()
|
||||
} else {
|
||||
session.publishable_origin_head_proxy_seq.into_iter().collect()
|
||||
};
|
||||
startup.revisions.get(&head).is_none_or(|revision| {
|
||||
mode != shared::model::HlsStartupMode::Conservative && !revision.revision().is_startup_head_ready(mode)
|
||||
if heads.is_empty() {
|
||||
return true;
|
||||
}
|
||||
heads.iter().any(|head| {
|
||||
startup.revisions.get(head).is_none_or(|revision| {
|
||||
mode != shared::model::HlsStartupMode::Conservative
|
||||
&& !revision.revision().is_startup_head_ready(mode)
|
||||
})
|
||||
})
|
||||
});
|
||||
let fetch_limit = if initial_barrier { 1 } else { policy.max_session_segment_fetches };
|
||||
@@ -404,6 +413,20 @@ impl HlsSegmentWorkerPool {
|
||||
let mut session = context.session.write().await;
|
||||
self.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, finished_at_ms)
|
||||
};
|
||||
if let Some(recovery_generation) = completion.recovery_generation {
|
||||
if let Some(recovery) = self.revision_recovery.get() {
|
||||
let session_id = context.session.read().await.proxy_session_id.clone();
|
||||
recovery
|
||||
.recover_segment_leases_after_fetch(
|
||||
&session_id,
|
||||
snapshot.proxy_seq,
|
||||
recovery_generation,
|
||||
finished_at_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
let mut notifier = completion.notifier;
|
||||
if completion.stale_commit_cleanup_reserved {
|
||||
if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await {
|
||||
|
||||
@@ -447,12 +447,13 @@ async fn bind_startup_revision(
|
||||
)));
|
||||
}
|
||||
let decides_policy = startup_fill_decides_policy(&session, snapshot.proxy_seq);
|
||||
revision.revision().bind_origin(&session);
|
||||
let Some(startup) = &mut session.startup else { return Ok(()) };
|
||||
startup.policy_fixed |= decides_policy;
|
||||
startup.revisions.insert(snapshot.proxy_seq, revision.clone());
|
||||
// An optimistic window may become complete when its last successor is
|
||||
// accepted, after the head already has bytes.
|
||||
if startup.config.mode.publishes_pending_successors() && session.last_rendered_manifest.is_none() {
|
||||
if startup.config.mode.publishes_pending_successors() {
|
||||
if let Err(error) = session.render_and_store_manifest(current_time_millis()) {
|
||||
debug!(
|
||||
"HLS optimistic startup render after successor binding failed: proxy_seq={} error={error:?}",
|
||||
@@ -467,6 +468,9 @@ async fn bind_startup_revision(
|
||||
/// fill may fix the session policy or fall back to `first_ready`. A successor
|
||||
/// without replay stays raw and is delivered once complete.
|
||||
fn startup_fill_decides_policy(session: &super::super::HlsSession, proxy_seq: u64) -> bool {
|
||||
if session.uses_fast_start_live_window() {
|
||||
return session.live_window_heads().contains(&proxy_seq);
|
||||
}
|
||||
!session.startup_mode().publishes_pending_successors()
|
||||
|| session.publishable_origin_head_proxy_seq == Some(proxy_seq)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use super::{
|
||||
mod admission;
|
||||
mod behavior;
|
||||
mod lifecycle;
|
||||
mod live_window;
|
||||
mod persistence;
|
||||
mod protocol;
|
||||
mod query;
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
use super::{
|
||||
fetch_context, install_startup, normal_manifest, spawn_segment_server, SegmentFetchPolicy, SegmentFetchPriority,
|
||||
};
|
||||
use crate::{
|
||||
api::{HlsLiveWindowView, HlsManifestRenderer},
|
||||
CachedSegmentMetadata, HlsSession, HlsSessionKey, SegmentCacheStatus, SegmentRevisionKind,
|
||||
};
|
||||
use shared::model::{HlsStartupMode, HlsStripMode};
|
||||
use std::{fmt::Write as _, sync::atomic::Ordering};
|
||||
use tuliprox_core::model::StripConfig;
|
||||
|
||||
const MODES: [HlsStartupMode; 3] =
|
||||
[HlsStartupMode::FirstReady, HlsStartupMode::Progressive, HlsStartupMode::Optimistic];
|
||||
|
||||
fn manifest(
|
||||
origin: &str,
|
||||
first: u64,
|
||||
durations: &[u64],
|
||||
) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest {
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-MEDIA-SEQUENCE:{first}\n");
|
||||
for (offset, duration) in durations.iter().enumerate() {
|
||||
writeln!(body, "#EXTINF:{duration},\n{origin}/{}.ts", first + u64::try_from(offset).unwrap()).unwrap();
|
||||
}
|
||||
normal_manifest(&body)
|
||||
}
|
||||
|
||||
fn prepare_selected(session: &mut HlsSession, sequences: &[u64], mode: HlsStartupMode, complete: bool) {
|
||||
let startup = session.startup.as_mut().unwrap();
|
||||
for (index, seq) in sequences.iter().enumerate() {
|
||||
let revision = startup
|
||||
.store
|
||||
.create(
|
||||
session.proxy_session_id.clone(),
|
||||
*seq,
|
||||
if mode == HlsStartupMode::FirstReady {
|
||||
SegmentRevisionKind::Processed
|
||||
} else {
|
||||
SegmentRevisionKind::Raw
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
if complete || (mode == HlsStartupMode::FirstReady && index == 0) {
|
||||
revision.revision().complete(CachedSegmentMetadata { path: std::path::PathBuf::from("unused"), size: 1 });
|
||||
} else if index == 0 || mode != HlsStartupMode::Optimistic {
|
||||
revision.revision().prefix_available.store(1, Ordering::Release);
|
||||
}
|
||||
startup.revisions.insert(*seq, revision);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_matrix_renders_only_selected_revisions_and_selects_real_fetch_snapshots() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in MODES {
|
||||
for (count, strip_mode, strip, offset, expected) in [
|
||||
(6, HlsStripMode::Segments, 0, 0, vec![3, 4, 5]),
|
||||
(6, HlsStripMode::Segments, 1, 0, vec![2, 3, 4]),
|
||||
(6, HlsStripMode::Segments, 2, 0, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Segments, 3, 0, vec![0, 1, 2]),
|
||||
(6, HlsStripMode::Segments, u64::MAX, 0, vec![0, 1, 2]),
|
||||
(12, HlsStripMode::Segments, 1, 0, vec![8, 9, 10]),
|
||||
(12, HlsStripMode::Segments, 1, 9_000_000, vec![9_000_008, 9_000_009, 9_000_010]),
|
||||
(6, HlsStripMode::Seconds, 0, 0, vec![3, 4, 5]),
|
||||
(6, HlsStripMode::Seconds, 7, 0, vec![2, 3, 4]),
|
||||
(6, HlsStripMode::Seconds, 8, 0, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Seconds, 10, 0, vec![1, 2, 3]),
|
||||
(6, HlsStripMode::Seconds, u64::MAX, 0, vec![0, 1, 2]),
|
||||
] {
|
||||
for parallelism in [1, 2, 3] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy {
|
||||
max_session_segment_fetches: parallelism,
|
||||
max_prefetch_queue_depth: 0,
|
||||
..Default::default()
|
||||
};
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
state.proxy_next_seq = Some(offset);
|
||||
let durations = if count == 6 { vec![4, 4, 4, 4, 3, 7] } else { vec![4; count] };
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 100_000, &durations)).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: strip_mode, value: strip };
|
||||
state.configure_segment_prefetch_queue(0);
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
let plan = {
|
||||
let mut state = context.session.write().await;
|
||||
let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap();
|
||||
assert_eq!(plan.visible, expected, "mode={mode} strip={strip} offset={offset}");
|
||||
state.queue_manifest_fetch_candidates(10, false);
|
||||
assert_eq!(crate::renderer_candidate_window_proxy_seqs(&state), expected);
|
||||
let protected = crate::gc::ProtectedSet::from_session(&state);
|
||||
assert_eq!(
|
||||
protected.segment_proxy_seqs.into_iter().collect::<std::collections::BTreeSet<_>>(),
|
||||
expected.iter().copied().collect()
|
||||
);
|
||||
plan
|
||||
};
|
||||
// Exercise the real scheduler, including the entry barrier. Its
|
||||
// next fetch must be the chosen head, never an older origin URI.
|
||||
for expected_seq in &expected {
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap();
|
||||
assert_eq!(snapshot.proxy_seq, *expected_seq);
|
||||
assert_eq!(snapshot.origin_seq, 100_000 + expected_seq - offset);
|
||||
assert_eq!(snapshot.priority, SegmentFetchPriority::RenderWindow);
|
||||
let mut state = context.session.write().await;
|
||||
state.active_segment_fetches -= 1;
|
||||
prepare_selected(&mut state, &[*expected_seq], mode, true);
|
||||
}
|
||||
assert!(worker.next_fetch_snapshot(&context, 12, &policy).await.is_none());
|
||||
let mut state = context.session.write().await;
|
||||
prepare_selected(&mut state, &expected, mode, false);
|
||||
let rendered = HlsManifestRenderer::render_live_window(&state, &plan, 12).unwrap();
|
||||
assert_eq!(rendered.segment_proxy_seqs, expected);
|
||||
assert_eq!(state.startup.as_ref().unwrap().revisions.len(), 3);
|
||||
assert!(rendered.body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", expected[0])));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_short_empty_readiness_and_stale_plans_are_explicit() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in MODES {
|
||||
for count in 0..=3 {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy::default();
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 0, &vec![4; count])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Seconds, value: u64::MAX };
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
let mut state = context.session.write().await;
|
||||
state.record_live_window_origin_commit(0);
|
||||
let Some(plan) = state.plan_live_window(HlsLiveWindowView::Entry) else {
|
||||
assert_eq!(count, 0);
|
||||
continue;
|
||||
};
|
||||
assert_eq!(plan.visible.len(), count);
|
||||
assert_eq!(plan.effective_strip_segments, 0);
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err());
|
||||
prepare_selected(&mut state, &plan.visible, mode, false);
|
||||
// Qualification of a one/two-segment source is still mandatory.
|
||||
assert_eq!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok(), count == 3);
|
||||
state.origin_control.qualification = crate::short_manifest::HlsOriginQualification::Qualified {
|
||||
basis: crate::short_manifest::HlsOriginQualificationBasis::capture(&state),
|
||||
generation: crate::manifest_acceptance::HlsManifestAcceptanceGeneration(1),
|
||||
windows: crate::short_manifest::HlsQualifiedOriginWindows::ShortAllowed,
|
||||
publication_boundary: None,
|
||||
};
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok());
|
||||
state.activity.origin_work_generation += 1;
|
||||
assert!(!plan.is_current(&state));
|
||||
assert!(state.plan_live_window(HlsLiveWindowView::Entry).is_none());
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_http_fills_selected_head_with_existing_progressive_delivery() {
|
||||
assert_live_window_http_fill(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "manual probe: set TULIPROX_HLS_WINDOW_ORIGIN to a controlled streaming fixture"]
|
||||
async fn live_window_http_fills_selected_head_against_manual_origin() {
|
||||
let origin = std::env::var("TULIPROX_HLS_WINDOW_ORIGIN").expect("manual origin URL must be configured");
|
||||
assert_live_window_http_fill(Some(&origin)).await;
|
||||
}
|
||||
|
||||
async fn assert_live_window_http_fill(external: Option<&str>) {
|
||||
let server = spawn_segment_server(0).await;
|
||||
let base = external.unwrap_or(&server.base_url);
|
||||
for mode in MODES {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy { max_prefetch_queue_depth: 0, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
let origin = if external.is_some() {
|
||||
let url = format!("{base}/index.m3u8");
|
||||
let body =
|
||||
context.client.get(&url).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
|
||||
match tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(&body, &url) {
|
||||
tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) => manifest,
|
||||
tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::TransientPassthrough {
|
||||
..
|
||||
} => panic!("expected ordinary live origin"),
|
||||
}
|
||||
} else {
|
||||
manifest(base, 0, &[4; 6])
|
||||
};
|
||||
state.apply_origin_manifest(&origin).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
state.configure_segment_prefetch_queue(0);
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
context.session.write().await.startup.as_mut().unwrap().worker = std::sync::Weak::new();
|
||||
super::grant_usable_worker_access_lease_at(
|
||||
&worker,
|
||||
&context.session.read().await.proxy_session_id,
|
||||
tuliprox_core::utils::current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
context.session.write().await.queue_manifest_fetch_candidates(10, false);
|
||||
for seq in [2, 3, 4] {
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap();
|
||||
assert_eq!(snapshot.proxy_seq, seq);
|
||||
let task_context = context.clone();
|
||||
let task_snapshot = snapshot.clone();
|
||||
let task_policy = policy.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await
|
||||
});
|
||||
if external.is_some() && mode != HlsStartupMode::FirstReady {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
if context
|
||||
.session
|
||||
.read()
|
||||
.await
|
||||
.startup
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.revisions
|
||||
.get(&seq)
|
||||
.is_some_and(|revision| revision.revision().is_publishable())
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&seq].clone();
|
||||
let mut response =
|
||||
crate::progressive_startup::revision_body(revision, std::time::Duration::from_secs(5), 0, None);
|
||||
let prefix = futures::StreamExt::next(&mut response).await.unwrap().unwrap();
|
||||
assert!(prefix.starts_with(b"segment-"));
|
||||
assert!(prefix.len() < 12);
|
||||
assert!(!task.is_finished(), "raw prefix must arrive before held origin EOF");
|
||||
}
|
||||
let metadata = task.await.unwrap().unwrap();
|
||||
assert!(metadata.content_length > 0);
|
||||
let mut state = context.session.write().await;
|
||||
state.active_segment_fetches -= 1;
|
||||
state.segments.get_mut(&seq).unwrap().status =
|
||||
SegmentCacheStatus::Ready { content_length: metadata.content_length, ready_at_ms: 12 };
|
||||
}
|
||||
let state = context.session.read().await;
|
||||
let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap();
|
||||
assert_eq!(
|
||||
HlsManifestRenderer::render_live_window(&state, &plan, 12).unwrap().segment_proxy_seqs,
|
||||
vec![2, 3, 4]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_readiness_uses_selected_head_and_never_skips_a_failure() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in MODES {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy::default();
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
install_startup(&context, &worker, mode).await;
|
||||
let mut state = context.session.write().await;
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 1, &[4; 6])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
let plan = state.plan_live_window(HlsLiveWindowView::Entry).unwrap();
|
||||
let head = plan.visible[0];
|
||||
prepare_selected(&mut state, &plan.visible, mode, false);
|
||||
let startup = state.startup.as_mut().unwrap();
|
||||
let entry = startup.revisions.remove(&head).unwrap();
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err());
|
||||
state.startup.as_mut().unwrap().revisions.insert(head, entry);
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_ok());
|
||||
let successor = plan.visible[1];
|
||||
state.startup.as_ref().unwrap().revisions[&successor].revision().fail();
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err());
|
||||
state.segments.get_mut(&head).unwrap().status =
|
||||
SegmentCacheStatus::FailedPermanent { failed_at_ms: 0, status: Some(axum::http::StatusCode::NOT_FOUND) };
|
||||
let origin_head = state.publishable_origin_head_proxy_seq;
|
||||
super::super::recompute_unpublished_live_head(&mut state);
|
||||
assert_eq!(state.publishable_origin_head_proxy_seq, origin_head);
|
||||
assert_eq!(state.plan_live_window(HlsLiveWindowView::Entry).unwrap().visible, plan.visible);
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 2, &[4; 6])).unwrap();
|
||||
assert!(!plan.is_current(&state));
|
||||
assert!(HlsManifestRenderer::render_live_window(&state, &plan, 0).is_err());
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 8, &[])).unwrap();
|
||||
assert!(state.uses_fast_start_live_window());
|
||||
assert!(state.plan_live_window(HlsLiveWindowView::Entry).is_none());
|
||||
assert!(HlsManifestRenderer::render(&state, 0).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_active_successors_are_required_with_zero_reserve_and_demand_keeps_priority() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in MODES {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy =
|
||||
SegmentFetchPolicy { max_prefetch_queue_depth: 0, max_session_segment_fetches: 3, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
state.configure_segment_prefetch_queue(0);
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
prepare_selected(&mut state, &[2, 3, 4], mode, true);
|
||||
for seq in [2, 3, 4] {
|
||||
state.segments.get_mut(&seq).unwrap().status =
|
||||
SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 };
|
||||
}
|
||||
state.live_window_demands.insert(
|
||||
crate::HlsAccessLeaseId("active".into()),
|
||||
crate::live_window::HlsLiveWindowDemand {
|
||||
issued_at_ms: 0,
|
||||
view: HlsLiveWindowView::Active { first_published: 2 },
|
||||
published: vec![2, 3, 4],
|
||||
},
|
||||
);
|
||||
let report = state.queue_manifest_fetch_candidates(10, false);
|
||||
assert_eq!(report.render_window_queued, 1);
|
||||
assert_eq!(report.prefetch_queued, 0);
|
||||
assert!(state.queue_segment_fetch_candidate(0, SegmentFetchPriority::Demand, 10));
|
||||
}
|
||||
let demand = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap();
|
||||
assert_eq!((demand.proxy_seq, demand.priority), (0, SegmentFetchPriority::Demand));
|
||||
let successor = worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap();
|
||||
assert_eq!((successor.proxy_seq, successor.priority), (5, SegmentFetchPriority::RenderWindow));
|
||||
assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_entry_barrier_and_parallel_limits_follow_the_selected_head() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in MODES {
|
||||
for parallelism in [1, 2, 3] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy { max_session_segment_fetches: parallelism, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
context.session.write().await.queue_manifest_fetch_candidates(10, false);
|
||||
assert_eq!(worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap().proxy_seq, 2);
|
||||
if mode != HlsStartupMode::Optimistic {
|
||||
assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none());
|
||||
prepare_selected(&mut *context.session.write().await, &[2], mode, false);
|
||||
}
|
||||
for seq in 3..2 + u64::try_from(parallelism).unwrap() {
|
||||
assert_eq!(worker.next_fetch_snapshot(&context, 11, &policy).await.unwrap().proxy_seq, seq);
|
||||
}
|
||||
assert!(worker.next_fetch_snapshot(&context, 11, &policy).await.is_none());
|
||||
assert_eq!(context.session.read().await.active_segment_fetches, parallelism);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_window_raw_fallback_is_decided_by_the_selected_head_even_with_an_older_demand() {
|
||||
let server = spawn_segment_server(0).await;
|
||||
for mode in [HlsStartupMode::Progressive, HlsStartupMode::Optimistic] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let policy = SegmentFetchPolicy::default();
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
*state = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0);
|
||||
state.apply_origin_manifest(&manifest(&server.base_url, 0, &[4; 6])).unwrap();
|
||||
state.live_window_strip = StripConfig { mode: HlsStripMode::Segments, value: 1 };
|
||||
}
|
||||
install_startup(&context, &worker, mode).await;
|
||||
{
|
||||
let mut state = context.session.write().await;
|
||||
let startup = state.startup.as_mut().unwrap();
|
||||
startup.config.max_progressive_bytes_per_segment = shared::model::Bytes::new(4);
|
||||
startup.worker = std::sync::Weak::new();
|
||||
state.queue_manifest_fetch_candidates(10, false);
|
||||
state.queue_segment_fetch_candidate(0, SegmentFetchPriority::Demand, 10);
|
||||
}
|
||||
super::grant_usable_worker_access_lease_at(
|
||||
&worker,
|
||||
&context.session.read().await.proxy_session_id,
|
||||
tuliprox_core::utils::current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
for seq in [0, 2] {
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap();
|
||||
assert_eq!(snapshot.proxy_seq, seq);
|
||||
let result = super::super::fetch_segment_into_cache(&context, &snapshot, &policy).await;
|
||||
assert!(result.is_ok());
|
||||
let mut state = context.session.write().await;
|
||||
worker.apply_segment_fetch_result(&mut state, &snapshot, &policy, result, 11);
|
||||
if seq == 0 {
|
||||
assert_eq!(state.startup_mode(), mode, "an older demand cannot choose the entry fallback");
|
||||
assert!(!state.startup.as_ref().unwrap().policy_fixed);
|
||||
} else {
|
||||
assert_eq!(state.startup_mode(), HlsStartupMode::FirstReady);
|
||||
assert!(state.startup.as_ref().unwrap().policy_fixed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -250,3 +250,114 @@ async fn optimistic_successor_replay_rejection_keeps_the_session_policy() -> std
|
||||
assert!(prepared_head.raw);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optimistic_binding_renders_cold_and_retained_sessions_in_both_event_orders() -> std::io::Result<()> {
|
||||
for (retained, head_first) in [(false, false), (false, true), (true, false), (true, true)] {
|
||||
// All origin input is supplied in memory; this probe opens no HTTP connection.
|
||||
let server = super::TestSegmentServer {
|
||||
base_url: "http://127.0.0.1:9".into(),
|
||||
requests: Arc::new(tokio::sync::Mutex::new(Vec::new())),
|
||||
task: tokio::spawn(async {}),
|
||||
};
|
||||
let directory = tempfile::tempdir()?;
|
||||
let policy = SegmentFetchPolicy { max_session_segment_fetches: 3, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut session = context.session.write().await;
|
||||
if retained {
|
||||
for entry in session.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 };
|
||||
}
|
||||
session.render_and_store_manifest(1).unwrap();
|
||||
assert_eq!(session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, 1);
|
||||
}
|
||||
session.apply_origin_manifest(&super::normal_manifest(
|
||||
"#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4\n#EXTINF:4,\n4.ts\n#EXTINF:4,\n5.ts\n#EXTINF:4,\n6.ts\n"
|
||||
)).unwrap();
|
||||
while session.segment_prefetch_queue.pop_next().is_some() {}
|
||||
for entry in session.segments.values_mut() {
|
||||
if matches!(entry.status, SegmentCacheStatus::Queued { .. }) {
|
||||
entry.status = SegmentCacheStatus::Discovered;
|
||||
}
|
||||
}
|
||||
for seq in [4, 5, 6] {
|
||||
assert!(session.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Demand, 10));
|
||||
}
|
||||
}
|
||||
install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await;
|
||||
context.session.write().await.startup.as_mut().unwrap().worker = std::sync::Weak::new();
|
||||
let mut snapshots = std::collections::BTreeMap::new();
|
||||
for _ in 0..3 {
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap();
|
||||
snapshots.insert(snapshot.proxy_seq, snapshot);
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let head = super::super::prepare_startup_fill(&context, &snapshots[&4], Some(2), deadline)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(format!("{e:?}")))?;
|
||||
if head_first {
|
||||
head.observed.lock().await.observe(b"a").await?;
|
||||
}
|
||||
let mut successors = Vec::new();
|
||||
for seq in [5, 6] {
|
||||
let successor = super::super::prepare_startup_fill(&context, &snapshots[&seq], Some(2), deadline)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(format!("{e:?}")))?;
|
||||
successors.push(successor);
|
||||
}
|
||||
if !head_first {
|
||||
let session = context.session.read().await;
|
||||
assert_eq!(
|
||||
session.last_rendered_manifest.as_ref().map(|manifest| manifest.first_proxy_seq),
|
||||
retained.then_some(1)
|
||||
);
|
||||
drop(session);
|
||||
head.observed.lock().await.observe(b"a").await?;
|
||||
}
|
||||
{
|
||||
let session = context.session.read().await;
|
||||
assert_eq!(crate::HlsManifestRenderer::render(&session, 20).unwrap().first_proxy_seq, 4);
|
||||
let stored = session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq;
|
||||
assert_eq!(stored, 4, "retained={retained} head_first={head_first}");
|
||||
assert!(session
|
||||
.startup
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.revisions
|
||||
.values()
|
||||
.all(|owner| matches!(owner.revision().state(), crate::SegmentRevisionState::Pending)));
|
||||
}
|
||||
// A second body chunk is not another render event.
|
||||
let rendered_at = context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms;
|
||||
head.observed.lock().await.observe(b"b").await?;
|
||||
assert_eq!(context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms, rendered_at);
|
||||
assert!(server.requests.lock().await.is_empty());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retired_generation_cannot_bind_or_render_an_optimistic_revision() -> std::io::Result<()> {
|
||||
let server = spawn_sequence_response_server(Vec::new()).await;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let policy = SegmentFetchPolicy::default();
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await;
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.expect("snapshot");
|
||||
context.session.write().await.activity.origin_work_generation += 1;
|
||||
let result = super::super::prepare_startup_fill(
|
||||
&context,
|
||||
&snapshot,
|
||||
Some(2),
|
||||
tokio::time::Instant::now() + Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
let session = context.session.read().await;
|
||||
assert!(session.startup.as_ref().expect("startup").revisions.is_empty());
|
||||
assert!(session.last_rendered_manifest.is_none());
|
||||
assert_eq!(context.segment_cache.remaining_revision_bytes(&session.proxy_session_id), 0);
|
||||
assert_eq!(super::temp_cache_files(directory.path()).len(), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -11,20 +11,31 @@ async fn startup_failure_and_cancellation_close_each_commit_transition() -> std:
|
||||
StartupWritePhase::AfterPublication,
|
||||
] {
|
||||
for cancel in [false, true] {
|
||||
startup_transition_round(phase, cancel).await?;
|
||||
for head in [1, 3] {
|
||||
startup_transition_round(phase, cancel, head).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std::io::Result<()> {
|
||||
async fn startup_transition_round(phase: StartupWritePhase, cancel: bool, head: u64) -> std::io::Result<()> {
|
||||
let server = spawn_segment_server(0).await;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
if head == 3 {
|
||||
let mut body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string();
|
||||
for seq in 1..=6 {
|
||||
writeln!(body, "#EXTINF:4,\n{}/{seq}.ts", server.base_url).unwrap();
|
||||
}
|
||||
let mut session = context.session.write().await;
|
||||
session.apply_origin_manifest(&normal_manifest(&body)).unwrap();
|
||||
session.live_window_strip.value = 1;
|
||||
}
|
||||
install_startup(&context, &worker, shared::model::HlsStartupMode::FirstReady).await;
|
||||
clear_scheduled_prefetch(&context, &policy).await;
|
||||
context.session.write().await.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 10);
|
||||
context.session.write().await.queue_segment_fetch_candidate(head, SegmentFetchPriority::Demand, 10);
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap();
|
||||
let (entered, resume) = context.segment_cache.pause_startup_write(phase);
|
||||
let task_context = context.clone();
|
||||
@@ -34,7 +45,7 @@ async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std
|
||||
super::super::fetch_segment_into_cache(&task_context, &task_snapshot, &task_policy).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), entered).await.unwrap().unwrap();
|
||||
let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&1].clone();
|
||||
let revision = context.session.read().await.startup.as_ref().unwrap().revisions[&head].clone();
|
||||
if cancel {
|
||||
task.abort();
|
||||
assert!(matches!(task.await, Err(error) if error.is_cancelled()));
|
||||
@@ -68,7 +79,7 @@ async fn startup_transition_round(phase: StartupWritePhase, cancel: bool) -> std
|
||||
if phase == StartupWritePhase::AfterPublication {
|
||||
for _ in 0..2 {
|
||||
let metadata = revision.revision().wait_complete(tokio::time::Instant::now()).await?;
|
||||
assert_eq!(tokio::fs::read(metadata.path).await?, b"body:/1.ts");
|
||||
assert_eq!(tokio::fs::read(metadata.path).await?, format!("body:/{head}.ts").as_bytes());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -58,10 +58,23 @@ pub struct SegmentRevision {
|
||||
pub prefix_available: AtomicU64,
|
||||
state: watch::Sender<SegmentRevisionState>,
|
||||
exposed: AtomicBool,
|
||||
origin_binding: Mutex<Option<RevisionOriginBinding>>,
|
||||
pub(crate) file_pin: Mutex<Option<super::HlsRevisionFilePin>>,
|
||||
pub(crate) replay: Mutex<Option<super::progressive_startup::ProgressiveReplay>>,
|
||||
}
|
||||
|
||||
/// The origin mapping which produced these bytes, independent of the attempt's
|
||||
/// mutable Fetching/Ready status. A retained startup map alone is not proof that
|
||||
/// a revision still belongs to the session's current origin generation.
|
||||
#[derive(Debug, Clone)]
|
||||
struct RevisionOriginBinding {
|
||||
generation: u64,
|
||||
origin_key: crate::OriginSegmentKey,
|
||||
cache_key: SegmentCacheKey,
|
||||
fetch_ref: Option<crate::OriginSegmentFetchRef>,
|
||||
encryption: Option<crate::HlsSegmentEncryption>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for SegmentRevision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("SegmentRevision")
|
||||
@@ -74,6 +87,32 @@ impl fmt::Debug for SegmentRevision {
|
||||
|
||||
impl SegmentRevision {
|
||||
pub fn id(&self) -> SegmentRevisionId { self.key.id }
|
||||
pub(crate) fn was_exposed(&self) -> bool { self.exposed.load(Ordering::Acquire) }
|
||||
|
||||
pub(crate) fn bind_origin(&self, session: &crate::HlsSession) {
|
||||
if let Some(entry) = session.segments.get(&self.key.proxy_seq) {
|
||||
*self.origin_binding.lock_unpoisoned() = Some(RevisionOriginBinding {
|
||||
generation: session.activity.origin_work_generation,
|
||||
origin_key: entry.origin_key,
|
||||
cache_key: entry.cache_key.clone(),
|
||||
fetch_ref: entry.origin_fetch_ref.clone(),
|
||||
encryption: entry.encryption.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn origin_binding_is_current(&self, session: &crate::HlsSession) -> bool {
|
||||
self.key.proxy_session_id() == &session.proxy_session_id
|
||||
&& self.origin_binding.lock_unpoisoned().as_ref().is_some_and(|binding| {
|
||||
binding.generation == session.activity.origin_work_generation
|
||||
&& session.segments.get(&self.key.proxy_seq).is_some_and(|entry| {
|
||||
entry.origin_key == binding.origin_key
|
||||
&& entry.cache_key == binding.cache_key
|
||||
&& entry.origin_fetch_ref == binding.fetch_ref
|
||||
&& entry.encryption == binding.encryption
|
||||
})
|
||||
})
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pin_count(&self) -> usize { self.pins.load(Ordering::Acquire) }
|
||||
|
||||
@@ -117,13 +156,12 @@ impl SegmentRevision {
|
||||
exposed
|
||||
}
|
||||
|
||||
/// An optimistic successor is listed before its body arrives. If it fails
|
||||
/// before receiving any byte, a new attempt may take over its URI.
|
||||
/// Only client exposure makes failed bytes immutable. Internally buffered
|
||||
/// prefix bytes do not prevent an optimistic retry from taking over the URI.
|
||||
pub(crate) fn is_replaceable_in(&self, mode: HlsStartupMode) -> bool {
|
||||
mode.publishes_pending_successors()
|
||||
&& matches!(*self.state.borrow(), SegmentRevisionState::Failed)
|
||||
&& !self.exposed.load(Ordering::Acquire)
|
||||
&& self.prefix_available.load(Ordering::Acquire) == 0
|
||||
}
|
||||
|
||||
pub(crate) fn replay_exhausted(&self) {
|
||||
@@ -164,8 +202,9 @@ impl SegmentRevision {
|
||||
}
|
||||
|
||||
/// Whether this revision may be listed in a startup window. Optimistic
|
||||
/// startup lists a successor as soon as the origin accepted it; delivery
|
||||
/// of a successor always waits for its complete body.
|
||||
/// startup lists a successor as soon as the origin accepted it. The lease's
|
||||
/// first full request may stream it once replay bytes arrive; subsequent
|
||||
/// requests wait for its complete body.
|
||||
pub fn is_window_publishable(&self, mode: HlsStartupMode) -> bool {
|
||||
if mode.publishes_pending_successors() {
|
||||
!matches!(*self.state.borrow(), SegmentRevisionState::Failed)
|
||||
@@ -261,6 +300,7 @@ impl SegmentRevisionStore {
|
||||
prefix_available: AtomicU64::new(0),
|
||||
state: watch::channel(SegmentRevisionState::Pending).0,
|
||||
exposed: AtomicBool::new(false),
|
||||
origin_binding: Mutex::new(None),
|
||||
file_pin: Mutex::new(None),
|
||||
replay: Mutex::new(None),
|
||||
});
|
||||
|
||||
@@ -279,6 +279,13 @@ pub struct HlsSession {
|
||||
pub origin_control: HlsSessionOriginControl,
|
||||
pub render_policy: RenderPolicy,
|
||||
pub startup: Option<super::HlsSessionStartup>,
|
||||
/// None before an origin commit; Some(empty) is an explicitly empty origin window.
|
||||
pub(crate) origin_window_proxy_seqs: Option<Vec<u64>>,
|
||||
pub(crate) origin_window_durations_ms: Vec<u64>,
|
||||
pub(crate) live_window_origin_commit_identity: Option<HlsManifestCommitIdentity>,
|
||||
pub(crate) live_window_origin_work_generation: Option<u64>,
|
||||
pub live_window_strip: tuliprox_core::model::StripConfig,
|
||||
pub(crate) live_window_demands: HashMap<super::HlsAccessLeaseId, super::live_window::HlsLiveWindowDemand>,
|
||||
pub last_rendered_manifest: Option<RenderedManifest>,
|
||||
manifest_commit_generation: u64,
|
||||
/// Origin manifests committed into this session.
|
||||
@@ -359,6 +366,15 @@ impl HlsSession {
|
||||
last_effective_manifest_host: None,
|
||||
origin_refresh: OriginRefreshState::default(),
|
||||
origin_control: HlsSessionOriginControl::default(),
|
||||
origin_window_proxy_seqs: None,
|
||||
origin_window_durations_ms: Vec::new(),
|
||||
live_window_origin_commit_identity: None,
|
||||
live_window_origin_work_generation: None,
|
||||
live_window_strip: tuliprox_core::model::StripConfig {
|
||||
mode: shared::model::HlsStripMode::Segments,
|
||||
value: 0,
|
||||
},
|
||||
live_window_demands: HashMap::new(),
|
||||
render_policy: RenderPolicy::default(),
|
||||
startup: None,
|
||||
last_rendered_manifest: None,
|
||||
@@ -428,6 +444,16 @@ impl HlsSession {
|
||||
self.origin_manifest_commits = self.origin_manifest_commits.saturating_add(1);
|
||||
}
|
||||
|
||||
pub(crate) fn record_live_window_origin_commit(&mut self, now_ms: u64) {
|
||||
if self.uses_fast_start_live_window() {
|
||||
if let Some(identity) = self.next_manifest_commit_identity(now_ms) {
|
||||
self.manifest_commit_generation = identity.commit_generation();
|
||||
self.live_window_origin_commit_identity = Some(identity);
|
||||
self.live_window_origin_work_generation = Some(self.activity.origin_work_generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the committed manifest was stored after an origin commit newer
|
||||
/// than `origin_manifest_commits`. Render time alone cannot prove this:
|
||||
/// a segment completion may still re-render the previous origin window.
|
||||
@@ -916,6 +942,12 @@ impl fmt::Debug for HlsSession {
|
||||
.field("last_normal_manifest_commit_identity", &self.last_normal_manifest_commit_identity)
|
||||
.field("published_live_origin_baseline", &self.published_live_origin_baseline)
|
||||
.field("longest_rendered_playlist_duration_ms", &self.longest_rendered_playlist_duration_ms)
|
||||
.field("origin_window_proxy_seqs", &self.origin_window_proxy_seqs)
|
||||
.field("origin_window_durations_ms", &self.origin_window_durations_ms)
|
||||
.field("live_window_origin_commit_identity", &self.live_window_origin_commit_identity)
|
||||
.field("live_window_origin_work_generation", &self.live_window_origin_work_generation)
|
||||
.field("live_window_strip", &self.live_window_strip)
|
||||
.field("live_window_demands", &self.live_window_demands)
|
||||
.field("initial_prefetch_gap_segments", &self.initial_prefetch_gap_segments)
|
||||
.field("segment_prefetch_queue_len", &self.segment_prefetch_queue.len())
|
||||
.field("active_segment_fetches", &self.active_segment_fetches)
|
||||
|
||||
@@ -20,6 +20,7 @@ pub struct HlsSessionStartup {
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq)]
|
||||
pub struct HlsManifestRevisions {
|
||||
pub window_plan: Option<crate::live_window::HlsLiveWindowPublication>,
|
||||
pub mode: HlsStartupMode,
|
||||
pub retained_start_seq: u64,
|
||||
pub revisions: BTreeMap<u64, SegmentRevisionGuard>,
|
||||
|
||||
@@ -335,6 +335,7 @@ struct TimelineDraft {
|
||||
origin_version: Option<u16>,
|
||||
target_duration: Option<u32>,
|
||||
independent_segments: bool,
|
||||
origin_window_proxy_seqs: Vec<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
@@ -383,6 +384,7 @@ impl From<&HlsSession> for TimelineDraft {
|
||||
origin_version: session.origin_version,
|
||||
target_duration: session.target_duration,
|
||||
independent_segments: session.independent_segments,
|
||||
origin_window_proxy_seqs: session.live_origin_sequences().to_vec(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -502,6 +504,12 @@ impl HlsSession {
|
||||
self.origin_version = draft.origin_version;
|
||||
self.target_duration = draft.target_duration;
|
||||
self.independent_segments = draft.independent_segments;
|
||||
self.origin_window_durations_ms = draft
|
||||
.origin_window_proxy_seqs
|
||||
.iter()
|
||||
.filter_map(|seq| self.segments.get(seq).map(|entry| entry.duration_ms))
|
||||
.collect();
|
||||
self.origin_window_proxy_seqs = Some(draft.origin_window_proxy_seqs);
|
||||
}
|
||||
|
||||
/// Captures the contiguous timeline beginning at the lease's first advertised sequence.
|
||||
@@ -591,6 +599,7 @@ impl TimelineDraft {
|
||||
mode: TimelineApplyMode,
|
||||
) -> Result<TimelineApplyResult, TimelineMapError> {
|
||||
let candidate_plan = self.candidate_timeline_plan(manifest, effective_host_id, mode)?;
|
||||
self.origin_window_proxy_seqs.clear();
|
||||
if candidate_plan.segments.is_empty() {
|
||||
return Ok(TimelineApplyResult::default());
|
||||
}
|
||||
@@ -644,6 +653,7 @@ impl TimelineDraft {
|
||||
|
||||
self.publishable_origin_head_proxy_seq = handoff_publishable_head_proxy_seq.or(manifest_head_proxy_seq);
|
||||
self.publishable_origin_tail_proxy_seq = manifest_tail_proxy_seq;
|
||||
self.origin_window_proxy_seqs.clone_from(&result.segment_proxy_seqs);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
|
||||
@@ -268,6 +268,24 @@ Changing `rewrite_secret` is also disruptive for public HLS identifiers. Existin
|
||||
READY reserve, cache delivery and repair behavior. `first_ready` lowers initial admission to one processed start
|
||||
segment. Every listed successor must already have a successful HTTP 200 response and a nonempty body prefix.
|
||||
|
||||
For untagged, rolling live playlists, each new access lease in `first_ready`, `progressive` or `optimistic`
|
||||
selects its own three-segment entry window from the newest end of the current origin playlist. This also applies
|
||||
when joining an already running shared session. `hls_cache.strip` is evaluated before selecting that window:
|
||||
segment mode counts backwards from the live edge; seconds mode sums the actual `EXTINF` durations backwards.
|
||||
The holdback is capped so at least three segments remain when the origin provides three or more. For origin
|
||||
segments `0–5`, strip `0` starts at `3,4,5`; strip `1` starts at `2,3,4`. These three segments are fetched in
|
||||
chronological order with `RenderWindow` priority. Held-back newer segments are optional reserve prefetch.
|
||||
|
||||
After activation, strip no longer applies to that lease. Its window grows forwards up to six entries, without
|
||||
prepending segments before its previously advertised first sequence: `2,3,4` becomes `2–5`, then `2–6`, then
|
||||
`2–7`. Needed successors remain required fetch work even when reserve prefetch is disabled. Active slides retain
|
||||
contiguous cached predecessors from the lease's previous window when needed to preserve three target durations.
|
||||
A second lease independently selects its own three-segment entry. An idle re-entry also selects an entry
|
||||
window, while retaining that lease's progressive GET allowance and monotonic media sequence. A change that
|
||||
would move its sequence backwards keeps its previous valid view or returns the existing temporary error.
|
||||
The planned view is never stripped again during lease materialization. These rules do not change conservative,
|
||||
archive, ENDLIST, EVENT, VOD or other protected/transient playlist handling, or short-origin qualification.
|
||||
|
||||
```yaml
|
||||
reverse_proxy:
|
||||
hls_cache:
|
||||
@@ -280,17 +298,30 @@ reverse_proxy:
|
||||
max_deferred_repairs: 32
|
||||
```
|
||||
|
||||
`progressive` lets the first full segment request consume the existing shared origin fill before origin EOF.
|
||||
`progressive` lets each access lease's first authorized full segment GET consume the existing shared origin
|
||||
fill before origin EOF, regardless of its position in the published playlist. A full request has no Range header
|
||||
or uses `Range: bytes=0-`; HEAD and partial ranges wait for complete metadata/bytes and do not consume this
|
||||
opportunity. The first full request reserves the opportunity before waiting, even if its revision is already
|
||||
complete. Concurrent or subsequent requests, including retries, wait for complete bytes. A cancelled first GET
|
||||
does not re-arm the opportunity. A new access lease has its own opportunity; playlist refreshes and seeks do not
|
||||
reset an existing lease's allowance.
|
||||
The complete session delivers immutable raw TS revisions, including subsequent requests, retries and ranges.
|
||||
Repair and the corruption watchdog continue on the processed cache copy. Raw delivery does not correct source
|
||||
timestamp or transport-stream defects; prefer `first_ready` when a source needs repair.
|
||||
|
||||
`optimistic` behaves like `progressive` and publishes the initial playlist earlier. The head segment needs a
|
||||
body prefix, while listed successors only need an accepted HTTP 200 origin response. Successor fetches start
|
||||
together with the head, within the configured per-session concurrency. A client request for a successor waits
|
||||
for its complete body. If the origin fails a successor after it was listed, only that segment fails for the
|
||||
client; already published media never changes. Replay or disk fallback switches the session to `first_ready`
|
||||
exactly like `progressive`.
|
||||
body prefix, while listed successors only need an accepted HTTP 200 origin response. This head requirement
|
||||
controls playlist publication; it does not determine which segment the player requests first. Successor fetches start
|
||||
together with the head, within the configured per-session concurrency. The lease's first full request may select
|
||||
any published successor. It waits for a replay prefix, then streams before EOF; without replay capacity it waits
|
||||
for the complete revision instead. Other requests wait for complete bodies. If the origin fails after a revision
|
||||
has been exposed, the body errors and recovery requires a new playback entry; bytes already exposed are never
|
||||
silently replaced. A failed unexposed revision can recover through its successful retry. Existing replay and disk
|
||||
fallback rules remain in effect; successors do not change the head's session policy.
|
||||
|
||||
A revision is considered exposed when a progressive response is prepared, before the first body chunk is sent.
|
||||
This conservative boundary applies across leases sharing that revision, even if the first client disconnects
|
||||
before reading any bytes. Listing a revision in a manifest alone does not expose it.
|
||||
|
||||
Fast startup is limited to live, clear MPEG-TS objects without MAP or origin byte ranges. Archive, encrypted,
|
||||
fragmented-MP4 and transient sessions use their existing path. Before publication an incompatible source can fall back to that path.
|
||||
|
||||
@@ -158,12 +158,27 @@ Fix:
|
||||
A `503` with `Retry-After` can be normal during short origin/cache waits, especially near startup or when a requested
|
||||
segment is known but not ready yet.
|
||||
|
||||
An optimistic segment whose origin download failed before any bytes were exposed to a client stays temporarily
|
||||
unavailable while its retry chain is active. Its existing URI returns `503` with `Retry-After` during that wait,
|
||||
instead of a permanent `404`. After a successful retry, the same lease and URI serve the complete replacement;
|
||||
the player does not need to refresh the playlist first. Internally buffered prefix bytes do not count as exposure.
|
||||
If a failed revision was already exposed, Tuliprox ends only the affected lease and its old routes return `404`.
|
||||
A new playback entry can use the shared session. Completed revisions and already exposed bytes are never replaced.
|
||||
The first authorized full segment GET of each lease can claim progressive delivery, including for a published
|
||||
successor. HEAD and partial ranges do not consume that opportunity; subsequent GETs wait for complete bytes.
|
||||
See [startup modes](./shared-hls-configuration.md#optional-startup-modes) for the shared policy.
|
||||
|
||||
The degraded-origin admission check still applies: a retryable manifest failure disables accelerated admission,
|
||||
and an unchanged successful manifest does not clear that condition. Media progress is required to clear it.
|
||||
|
||||
Investigate when it repeats for too long:
|
||||
|
||||
| Log signal | Interpretation |
|
||||
| :--- | :--- |
|
||||
| `HLS segment demand fetch skipped by backpressure` | The session or global fetch limit is saturated. |
|
||||
| `HLS segment temporary failure counted` | The origin segment fetch is failing temporarily. |
|
||||
| `HLS revision recovery` | `replacement_bound` repairs an unexposed failed binding; `reentry_required` ends that exact lease. Includes anonymized identity, revision, sequence, generation, prefix size, and exposure. |
|
||||
| `HLS playlist wait` | A changed wait reason: `fresh_render_required`, `missing_revision_or_head_readiness`, or `admission_rejected`. Includes head readiness and degraded-origin state; repeated identical polls are not logged. |
|
||||
| `HLS manifest marked fresh-commit required after hard fetch failure` | Manifest refresh had a hard failure and needs a fresh commit. |
|
||||
| `HLS terminal manifest failed closed` | A typed lease-local terminal decision could not safely serve live or terminal media. The reason identifies state, bundle, deadline, capacity, or retry failure. |
|
||||
| `HLS terminal commit retry owner failed closed` | The bounded autonomous commit worker reached a typed terminal failure; inspect the reason and preceding recovery/bundle events. |
|
||||
|
||||
Reference in New Issue
Block a user