mirror of
https://github.com/euzu/tuliprox.git
synced 2026-10-31 11:12:36 +01:00
fix (hls): recover shared HLS retries and stream the first lease segment
This commit is contained in:
@@ -273,8 +273,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
server_path,
|
||||
bandwidth_learning,
|
||||
);
|
||||
let mut last_wait_reason = None;
|
||||
loop {
|
||||
let cached = read_hls_cached_manifest(session, options, started_at_ms).await;
|
||||
if cached.wait_for_initial_commit || (cached.rendered_body.is_none() && cached.transient_body.is_none()) {
|
||||
log_hls_manifest_wait_transition(&mut last_wait_reason, "fresh_render_required", session, access_lease_id)
|
||||
.await;
|
||||
}
|
||||
if !cached.wait_for_initial_commit {
|
||||
let prepared = if let Some(transient) = cached.transient_body {
|
||||
let materialized = view.materialize(&transient.body, "transient", transient.window_policy);
|
||||
@@ -342,6 +347,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
if access_lease_state != HlsAccessLeaseState::Pending {
|
||||
return None;
|
||||
}
|
||||
log_hls_manifest_wait_transition(
|
||||
&mut last_wait_reason,
|
||||
"missing_revision_or_head_readiness",
|
||||
session,
|
||||
access_lease_id,
|
||||
)
|
||||
.await;
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
@@ -356,6 +368,13 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response
|
||||
&& !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms)
|
||||
.await
|
||||
{
|
||||
log_hls_manifest_wait_transition(
|
||||
&mut last_wait_reason,
|
||||
"admission_rejected",
|
||||
session,
|
||||
access_lease_id,
|
||||
)
|
||||
.await;
|
||||
if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await {
|
||||
continue;
|
||||
}
|
||||
@@ -426,3 +445,28 @@ pub(in crate::api::endpoints::hls_api) async fn wait_for_hls_startup_evidence(
|
||||
tokio::time::sleep(wait_timeout.saturating_sub(elapsed).min(Duration::from_millis(25))).await;
|
||||
true
|
||||
}
|
||||
|
||||
/// Per-request state prevents a 25-ms polling loop from repeating diagnostics.
|
||||
async fn log_hls_manifest_wait_transition(
|
||||
previous: &mut Option<(&'static str, bool, bool)>,
|
||||
reason: &'static str,
|
||||
session: &HlsSessionHandle,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
) {
|
||||
let session = session.read().await;
|
||||
let degraded = session.origin_control.path_condition.is_degraded();
|
||||
let head = session.publishable_origin_head_proxy_seq.and_then(|seq| session.startup.as_ref()?.revisions.get(&seq));
|
||||
let head_ready = head.is_some_and(|head| head.revision().is_startup_head_ready(session.startup_mode()));
|
||||
let state = (reason, degraded, head_ready);
|
||||
if *previous == Some(state) {
|
||||
return;
|
||||
}
|
||||
*previous = Some(state);
|
||||
debug!(
|
||||
"HLS playlist wait: lease={} proxy_session={} reason={reason} degraded={degraded} head_ready={head_ready} origin_generation={} readiness_generation={} head_revision={:?} head_prefix={}",
|
||||
safe_hls_access_lease_id(lease_id), safe_proxy_session_id(&session.proxy_session_id),
|
||||
session.activity.origin_work_generation, session.activity.media_readiness_generation,
|
||||
head.map(|head| head.revision().id()),
|
||||
head.map_or(0, |head| head.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment(
|
||||
fingerprint: Fingerprint,
|
||||
axum::extract::Path(params): axum::extract::Path<HlsProxySegmentPathParams>,
|
||||
axum::extract::State(app_state): axum::extract::State<Arc<AppState>>,
|
||||
method: Method,
|
||||
headers: HeaderMap,
|
||||
) -> axum::response::Response {
|
||||
let proxy_session_id = ProxySessionId(params.proxy_session_id);
|
||||
@@ -83,6 +84,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment(
|
||||
&access_context,
|
||||
&fingerprint,
|
||||
&headers,
|
||||
&method,
|
||||
segment_file,
|
||||
now_ms,
|
||||
)
|
||||
@@ -162,12 +164,17 @@ pub(in crate::api::endpoints::hls_api) async fn demand_fetch_hls_live_segment(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_arguments,
|
||||
reason = "Retains request method and headers alongside the validated lease context."
|
||||
)]
|
||||
pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_lease(
|
||||
app_state: &Arc<AppState>,
|
||||
session: &HlsSessionHandle,
|
||||
access_context: &HlsAccessContext,
|
||||
fingerprint: &Fingerprint,
|
||||
headers: &HeaderMap,
|
||||
method: &Method,
|
||||
segment_file: HlsSegmentFile,
|
||||
now_ms: u64,
|
||||
) -> axum::response::Response {
|
||||
@@ -181,7 +188,9 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_le
|
||||
let Some(lease_identity) = current_lease.media_identity() else {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
};
|
||||
let response_context = hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await;
|
||||
let mut response_context =
|
||||
hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await;
|
||||
response_context.request_method = method.clone();
|
||||
let response = hls_resource_serve_outcome_response(
|
||||
app_state,
|
||||
access_context,
|
||||
|
||||
@@ -91,6 +91,7 @@ mod headers;
|
||||
mod late_custom_response;
|
||||
mod lease_lifecycle;
|
||||
mod manifest_entry_freshness;
|
||||
mod manifest_revision_recovery;
|
||||
mod manifest_short_window;
|
||||
mod manifest_startup;
|
||||
mod observability;
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
use super::{normal_manifest, response_body, test_app_state_with_hls_proxy, try_test_hls_cached_manifest_response};
|
||||
use crate::api::model::{
|
||||
HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey,
|
||||
SegmentCacheStatus,
|
||||
};
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use shared::model::{HlsCacheConfigDto, HlsStartupMode};
|
||||
use std::{fmt::Write as _, sync::Arc, time::Duration};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tuliprox_core::{
|
||||
model::{HlsCacheConfig, StripConfig},
|
||||
utils::current_time_millis,
|
||||
};
|
||||
use tuliprox_hls::api::SegmentFetchContext;
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::too_many_lines, reason = "One deterministic held-body scheduling and HTTP publication scenario.")]
|
||||
async fn retained_playlist_request_is_released_by_successor_headers_while_bodies_stay_open() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("origin");
|
||||
let origin = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (headers, header_gate) = tokio::sync::watch::channel(false);
|
||||
let (bodies, body_gate) = tokio::sync::watch::channel(false);
|
||||
let origin_task = tokio::spawn(async move {
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
socket = listener.accept() => {
|
||||
let (socket, _) = socket.unwrap();
|
||||
let mut headers = header_gate.clone();
|
||||
let mut bodies = body_gate.clone();
|
||||
children.spawn(async move {
|
||||
let mut socket = BufReader::new(socket);
|
||||
let mut request = String::new();
|
||||
socket.read_line(&mut request).await.unwrap();
|
||||
let head = request.contains("/103.ts ");
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
socket.read_line(&mut line).await.unwrap();
|
||||
if line == "\r\n" || line.is_empty() { break; }
|
||||
}
|
||||
let mut socket = socket.into_inner();
|
||||
if !head {
|
||||
while !*headers.borrow_and_update() { headers.changed().await.unwrap(); }
|
||||
}
|
||||
socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\n").await.unwrap();
|
||||
if head { socket.write_all(b"segment-").await.unwrap(); }
|
||||
while !*bodies.borrow_and_update() { bodies.changed().await.unwrap(); }
|
||||
socket.write_all(if head { b"body".as_slice() } else { b"segment-body".as_slice() }).await.unwrap();
|
||||
});
|
||||
}
|
||||
_ = children.join_next(), if !children.is_empty() => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut dto = HlsCacheConfigDto {
|
||||
cache_path: Some(directory.path().to_string_lossy().into_owned()),
|
||||
max_concurrent_segment_fetches_per_session: 3,
|
||||
..Default::default()
|
||||
};
|
||||
dto.startup.mode = HlsStartupMode::Optimistic;
|
||||
let manager = Arc::new(HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&dto)));
|
||||
let app = test_app_state_with_hls_proxy(manager.clone());
|
||||
let session =
|
||||
manager.get_or_create_session(HlsSessionKey::new(1, "retained-render"), b"secret", current_time_millis()).await;
|
||||
let manifest = |first: u64| {
|
||||
let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n");
|
||||
for seq in first..first + 3 {
|
||||
writeln!(body, "#EXTINF:4,\n{origin}/{seq}.ts").unwrap();
|
||||
}
|
||||
normal_manifest(&body)
|
||||
};
|
||||
let old_rendered_at = current_time_millis().saturating_sub(1000);
|
||||
let (old_head, new_head) = {
|
||||
let mut session = session.write().await;
|
||||
let startup = session.startup.take();
|
||||
session.apply_origin_manifest(&manifest(100)).unwrap();
|
||||
for entry in session.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: old_rendered_at };
|
||||
}
|
||||
let old_head = session.render_and_store_manifest(old_rendered_at).unwrap().first_proxy_seq;
|
||||
session.apply_origin_manifest(&manifest(103)).unwrap();
|
||||
session.startup = startup;
|
||||
let new_head = session.publishable_origin_head_proxy_seq.unwrap();
|
||||
while session.segment_prefetch_queue.pop_next().is_some() {}
|
||||
let candidates = session
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, entry)| !matches!(entry.status, SegmentCacheStatus::Ready { .. }))
|
||||
.map(|(seq, _)| *seq)
|
||||
.collect::<Vec<_>>();
|
||||
for seq in candidates {
|
||||
session.segments.get_mut(&seq).unwrap().status = SegmentCacheStatus::Discovered;
|
||||
assert!(session.queue_segment_fetch_candidate(
|
||||
seq,
|
||||
tuliprox_hls::api::SegmentFetchPriority::Demand,
|
||||
current_time_millis()
|
||||
));
|
||||
}
|
||||
(old_head, new_head)
|
||||
};
|
||||
let session_id = session.read().await.proxy_session_id.clone();
|
||||
let lease_id = HlsAccessLeaseId("waiting-reused-session".into());
|
||||
manager
|
||||
.prepare_access_lease(HlsAccessLease::pending(
|
||||
lease_id.clone(),
|
||||
HlsPlaybackFamilyKey::new("test-user", "render-gate"),
|
||||
session_id,
|
||||
"test-user".into(),
|
||||
"render-session".into(),
|
||||
1,
|
||||
"retained-render".into(),
|
||||
1,
|
||||
current_time_millis(),
|
||||
60_000,
|
||||
))
|
||||
.await;
|
||||
let client = reqwest::Client::new();
|
||||
manager
|
||||
.segment_worker_pool()
|
||||
.wake_scheduler(
|
||||
SegmentFetchContext {
|
||||
session: session.clone(),
|
||||
segment_cache: manager.segment_cache().clone(),
|
||||
segment_repair: manager.segment_repair().clone(),
|
||||
repair_access_lease_id: None,
|
||||
headers: HeaderMap::new(),
|
||||
origin_provider_session_headers: HeaderMap::new(),
|
||||
client: client.clone(),
|
||||
no_redirect_client: client,
|
||||
use_manual_redirects: false,
|
||||
origin_io: None,
|
||||
},
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
if session
|
||||
.read()
|
||||
.await
|
||||
.startup
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.revisions
|
||||
.values()
|
||||
.any(|revision| revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire) > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("head prefix");
|
||||
assert_eq!(session.read().await.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, old_head);
|
||||
let request = {
|
||||
let app = app.clone();
|
||||
let session = session.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
tokio::spawn(async move {
|
||||
try_test_hls_cached_manifest_response(
|
||||
&app,
|
||||
&session,
|
||||
&lease_id,
|
||||
HlsAccessLeaseState::Pending,
|
||||
&StripConfig { mode: shared::model::HlsStripMode::Segments, value: 0 },
|
||||
None,
|
||||
super::HlsCachedManifestOptions::initial(Duration::from_secs(2))
|
||||
.requiring_newer_manifest(old_rendered_at),
|
||||
)
|
||||
.await
|
||||
})
|
||||
};
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
assert!(!request.is_finished());
|
||||
headers.send_replace(true);
|
||||
let response = tokio::time::timeout(Duration::from_secs(1), request).await.unwrap().unwrap().expect("playlist");
|
||||
let answered_at = current_time_millis();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
|
||||
assert!(body.contains(&format!("{new_head:06}.ts")));
|
||||
let state = session.read().await;
|
||||
let rendered = state.last_rendered_manifest.as_ref().unwrap();
|
||||
assert!(answered_at.saturating_sub(rendered.rendered_at_ms) <= 50, "release exceeded two poll intervals");
|
||||
assert!(state.startup.as_ref().unwrap().revisions.values().all(|revision| !revision.revision().is_complete()));
|
||||
assert!(!*bodies.borrow());
|
||||
drop(state);
|
||||
bodies.send_replace(true);
|
||||
origin_task.abort();
|
||||
}
|
||||
@@ -92,12 +92,6 @@ app_state_views! {
|
||||
app_config, recordings, events <- event_manager, http_clients, recording_capacity,
|
||||
}
|
||||
|
||||
/// The handles the HLS proxy needs: itself, plus provider allocation and
|
||||
/// session accounting.
|
||||
hls_ctx => crate::api::model::hls_cache::HlsCtx {
|
||||
app_config, hls_proxy <- hls.proxy, active_provider, connection_manager, active_users,
|
||||
}
|
||||
|
||||
/// The handles admission reads: it decides over connections and users.
|
||||
admission_ctx => tuliprox_session::admission::AdmissionCtx {
|
||||
app_config, active_users, connection_manager,
|
||||
@@ -116,3 +110,23 @@ app_state_views! {
|
||||
maintenance as std::sync::Arc<dyn tuliprox_core::model::MaintenanceGate>,
|
||||
}
|
||||
}
|
||||
|
||||
impl AppStateView for crate::api::model::hls_cache::HlsCtx {
|
||||
fn from_app_state(state: &AppState) -> Self {
|
||||
let ctx = Self {
|
||||
app_config: state.app_config.clone(),
|
||||
hls_proxy: state.hls.proxy.clone(),
|
||||
active_provider: state.active_provider.clone(),
|
||||
connection_manager: state.connection_manager.clone(),
|
||||
active_users: state.active_users.clone(),
|
||||
};
|
||||
ctx.attach_lease_user_accounting();
|
||||
ctx
|
||||
}
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
/// The HLS runtime and its authoritative session-accounting owner.
|
||||
#[must_use]
|
||||
pub fn hls_ctx(&self) -> crate::api::model::hls_cache::HlsCtx { self.view() }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
//! Real Tuliprox process + a streaming origin. No application HTTP client is stubbed.
|
||||
use std::{
|
||||
fmt::Write as _,
|
||||
path::PathBuf,
|
||||
process::Stdio,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
|
||||
net::TcpListener,
|
||||
sync::watch,
|
||||
};
|
||||
|
||||
const BODY: &[u8] =
|
||||
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"));
|
||||
|
||||
// A configured Postman member is shared between the process scenarios.
|
||||
static SCENARIO: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
struct Fixture {
|
||||
directory: tempfile::TempDir,
|
||||
base: String,
|
||||
process: tokio::process::Child,
|
||||
origin: tokio::task::JoinHandle<()>,
|
||||
release: watch::Sender<bool>,
|
||||
release_retry: watch::Sender<bool>,
|
||||
attempts: Arc<AtomicUsize>,
|
||||
_scenario: tokio::sync::MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.process.start_kill();
|
||||
self.origin.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
#[allow(clippy::too_many_lines, reason = "Keeps the isolated process and streaming origin fixture together.")]
|
||||
async fn start() -> Self {
|
||||
let scenario = SCENARIO.lock().await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let origin_base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let manifest_base =
|
||||
std::env::var("TULIPROX_HLS_RECOVERY_MANIFEST_BASE").unwrap_or_else(|_| origin_base.clone());
|
||||
if manifest_base != origin_base {
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{manifest_base}/configure"))
|
||||
.json(&serde_json::json!({"segmentOrigin": origin_base}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 204);
|
||||
}
|
||||
let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"recovery\" group-title=\"Test\",Recovery Channel\n{manifest_base}/index.m3u8\n");
|
||||
let mut manifest = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n".to_string();
|
||||
for seq in 100..106 {
|
||||
writeln!(manifest, "#EXTINF:4,\n{origin_base}/{seq}.ts").unwrap();
|
||||
}
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let (release, released) = watch::channel(false);
|
||||
let (release_retry, retry_released) = watch::channel(false);
|
||||
let counted = attempts.clone();
|
||||
let origin = tokio::spawn(async move {
|
||||
let mut children = tokio::task::JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
accepted = listener.accept() => {
|
||||
let (socket, _) = accepted.unwrap();
|
||||
let catalog = catalog.clone(); let manifest = manifest.clone();
|
||||
let attempts = counted.clone(); let mut released = released.clone();
|
||||
let mut retry_released = retry_released.clone();
|
||||
children.spawn(async move {
|
||||
let mut reader = BufReader::new(socket);
|
||||
let mut request = String::new();
|
||||
reader.read_line(&mut request).await.unwrap();
|
||||
let path = request.split_whitespace().nth(1).unwrap().to_owned();
|
||||
loop {
|
||||
let mut line = String::new(); reader.read_line(&mut line).await.unwrap();
|
||||
if line == "\r\n" || line.is_empty() { break; }
|
||||
}
|
||||
let mut socket = reader.into_inner();
|
||||
if path == "/catalog.m3u" || path == "/index.m3u8" {
|
||||
let body = if path == "/index.m3u8" { manifest } else { catalog };
|
||||
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/vnd.apple.mpegurl\r\nConnection: close\r\n\r\n", body.len());
|
||||
socket.write_all(headers.as_bytes()).await.unwrap();
|
||||
socket.write_all(body.as_bytes()).await.unwrap();
|
||||
return;
|
||||
}
|
||||
let failed_attempt = path == "/101.ts" && attempts.fetch_add(1, Ordering::AcqRel) == 0;
|
||||
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: video/mp2t\r\nConnection: close\r\n\r\n", BODY.len());
|
||||
socket.write_all(headers.as_bytes()).await.unwrap();
|
||||
if failed_attempt {
|
||||
socket.write_all(b"stale-prefix").await.unwrap();
|
||||
// Hold an incomplete 200 open beyond the configured body idle timeout.
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
return;
|
||||
}
|
||||
if path == "/101.ts" {
|
||||
while !*retry_released.borrow_and_update() {
|
||||
if retry_released.changed().await.is_err() { return; }
|
||||
}
|
||||
}
|
||||
socket.write_all(&BODY[..188]).await.unwrap();
|
||||
while !*released.borrow_and_update() {
|
||||
if released.changed().await.is_err() { return; }
|
||||
}
|
||||
let _ = socket.write_all(&BODY[188..]).await;
|
||||
});
|
||||
}
|
||||
_ = children.join_next(), if !children.is_empty() => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = reserved.local_addr().unwrap().port();
|
||||
drop(reserved);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let root = directory.path();
|
||||
for name in ["web", "storage", "backup"] {
|
||||
std::fs::create_dir(root.join(name)).unwrap();
|
||||
}
|
||||
let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n origin_segment_timeout_ms: 300\n max_concurrent_segment_fetches_per_session: 3\n initial_manifest_wait_timeout_secs: 3\n startup:\n mode: optimistic\n segment_repair:\n max_level: off\n", root.join("web").display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display());
|
||||
let sources = format!("inputs:\n - name: recovery-origin\n type: m3u\n url: {origin_base}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [recovery-origin]\n targets:\n - name: recovery\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n");
|
||||
let users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Recovery test\nuser:\n - target: recovery\n credentials:\n - username: viewer\n password: test-password\n proxy: reverse\n server: local\n status: Active\n max_connections: 1\n");
|
||||
for (name, body) in [("config.yml", config), ("source.yml", sources), ("api-proxy.yml", users)] {
|
||||
std::fs::write(root.join(name), body).unwrap();
|
||||
}
|
||||
let log = std::fs::File::create(root.join("sut.log")).unwrap();
|
||||
let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox"))
|
||||
.args(["-s", "-c", "config.yml", "-i", "source.yml", "-a", "api-proxy.yml"])
|
||||
.current_dir(root)
|
||||
.stdout(Stdio::from(log.try_clone().unwrap()))
|
||||
.stderr(Stdio::from(log))
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.unwrap();
|
||||
Self {
|
||||
directory,
|
||||
base: format!("http://127.0.0.1:{port}"),
|
||||
process,
|
||||
origin,
|
||||
release,
|
||||
release_retry,
|
||||
attempts,
|
||||
_scenario: scenario,
|
||||
}
|
||||
}
|
||||
|
||||
fn log(&self) -> String { std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap() }
|
||||
|
||||
async fn playback(&mut self, client: &reqwest::Client) -> String {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
assert!(self.process.try_wait().unwrap().is_none(), "SUT exited: {}", self.log());
|
||||
if let Ok(response) =
|
||||
client.get(format!("{}/m3u?username=viewer&password=test-password", self.base)).send().await
|
||||
{
|
||||
if let Ok(body) = response.text().await {
|
||||
if let Some(url) = body.lines().find(|line| line.starts_with("http")) {
|
||||
return url.to_owned();
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(tokio::time::Instant::now() < deadline, "catalog unavailable: {}", self.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_media(base: &str, body: &str, index: usize) -> String {
|
||||
let uri = body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).nth(index).expect("media URI");
|
||||
reqwest::Url::parse(base).unwrap().join(uri).unwrap().to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incomplete_origin_200_recovers_same_segment_uri_without_a_playlist_refresh() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap();
|
||||
assert!(master.status().is_success(), "entry: {}", fixture.log());
|
||||
let master_body = master.text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master_body, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(playlist.status(), 200, "startup: {}", fixture.log());
|
||||
let playlist = playlist.text().await.unwrap();
|
||||
let segment = resolve_media(&variant, &playlist, 1);
|
||||
assert_eq!(fixture.attempts.load(Ordering::Acquire), 1);
|
||||
// Exercise unexposed successor recovery after consuming this lease's one
|
||||
// progressive request. A successor selected as the first GET may now pipe
|
||||
// its prefix and therefore cannot silently switch bytes after a body drop.
|
||||
let head = client.get(resolve_media(&variant, &playlist, 0)).send().await.unwrap();
|
||||
assert_eq!(head.status(), 200, "head must stream before EOF: {}", fixture.log());
|
||||
fixture.release.send_replace(true);
|
||||
assert_eq!(head.bytes().await.unwrap().as_ref(), BODY);
|
||||
let waiting = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(waiting.status(), 503, "retry must be temporary: {}", fixture.log());
|
||||
assert!(waiting.headers().contains_key("retry-after"));
|
||||
// Make the pending response independent of the origin retry's timing.
|
||||
fixture.release_retry.send_replace(true);
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let response = client.get(&segment).send().await.unwrap();
|
||||
if response.status() == 200 {
|
||||
assert_eq!(response.bytes().await.unwrap().as_ref(), BODY);
|
||||
break;
|
||||
}
|
||||
assert_eq!(response.status(), 503, "same URI must not become 404 during recovery: {}", fixture.log());
|
||||
assert!(tokio::time::Instant::now() < deadline, "retry never recovered: {}", fixture.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
assert!(fixture.attempts.load(Ordering::Acquire) >= 2);
|
||||
let refreshed = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(refreshed.status(), 200, "recovered binding must not conflict: {}", fixture.log());
|
||||
assert!(refreshed.text().await.unwrap().contains(reqwest::Url::parse(&segment).unwrap().path()));
|
||||
for _ in 0..3 {
|
||||
let replay = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY);
|
||||
}
|
||||
assert_eq!(fixture.attempts.load(Ordering::Acquire), 2, "repaired GETs reuse the committed revision");
|
||||
assert!(fixture.log().contains("outcome=replacement_bound"));
|
||||
// Retain run evidence only when requested; test configuration and credentials are synthetic.
|
||||
if let Ok(destination) = std::env::var("TULIPROX_HLS_RECOVERY_LOG") {
|
||||
std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap();
|
||||
}
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_requested_successor_streams_before_origin_eof() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap().text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap();
|
||||
assert_eq!(playlist.status(), 200, "startup: {}", fixture.log());
|
||||
let playlist = playlist.text().await.unwrap();
|
||||
// Skip both the playlist head and the deliberately failing first successor.
|
||||
let segment = resolve_media(&variant, &playlist, 2);
|
||||
let mut response = tokio::time::timeout(Duration::from_secs(2), client.get(&segment).send())
|
||||
.await
|
||||
.expect("first successor GET must not wait for origin EOF")
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200, "successor: {}", fixture.log());
|
||||
assert!(!response.headers().contains_key("content-length"));
|
||||
let prefix = response.chunk().await.unwrap().expect("progressive prefix");
|
||||
assert_eq!(prefix.as_ref(), &BODY[..188]);
|
||||
assert!(!*fixture.release.borrow(), "origin EOF is still held");
|
||||
fixture.release.send_replace(true);
|
||||
fixture.release_retry.send_replace(true);
|
||||
let mut bytes = prefix.to_vec();
|
||||
while let Some(chunk) = response.chunk().await.unwrap() {
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
assert_eq!(bytes, BODY);
|
||||
let replay = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap().as_ref(), BODY);
|
||||
if let Ok(destination) = std::env::var("TULIPROX_HLS_FIRST_REQUEST_LOG") {
|
||||
std::fs::copy(fixture.directory.path().join("sut.log"), PathBuf::from(destination)).unwrap();
|
||||
}
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exposed_failure_allows_a_new_player_with_user_connection_limit_one() {
|
||||
let mut fixture = Fixture::start().await;
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let entry = fixture.playback(&client).await;
|
||||
let master = client.get(&entry).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
|
||||
let variant = resolve_media(&entry, &master, 0);
|
||||
let playlist = client.get(&variant).send().await.unwrap().error_for_status().unwrap().text().await.unwrap();
|
||||
let segment = resolve_media(&variant, &playlist, 1);
|
||||
// The first full GET deliberately selects the successor, exercising the
|
||||
// integrated piping policy and exposing this attempt's prefix.
|
||||
let exposed = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(exposed.status(), 200, "first successor GET must pipe: {}", fixture.log());
|
||||
fixture.release.send_replace(true);
|
||||
assert!(exposed.bytes().await.is_err(), "exposed truncated body must not become successful EOF");
|
||||
let ended = client.get(&segment).send().await.unwrap();
|
||||
assert_eq!(ended.status(), 404, "old lease must end: {}", fixture.log());
|
||||
fixture.release_retry.send_replace(true);
|
||||
let new_player =
|
||||
reqwest::Client::builder().user_agent("new-reentry-player").timeout(Duration::from_secs(10)).build().unwrap();
|
||||
let master = new_player.get(&entry).send().await.unwrap();
|
||||
assert_eq!(master.status(), 200, "counted user reservation blocked reentry: {}", fixture.log());
|
||||
let body = master.text().await.unwrap();
|
||||
assert!(body.contains("#EXT-X-STREAM-INF"), "new entry must reach live playback: {body}");
|
||||
let new_variant = resolve_media(&entry, &body, 0);
|
||||
assert_ne!(new_variant, variant);
|
||||
let previous_session = reqwest::Url::parse(&variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned();
|
||||
let current_session =
|
||||
reqwest::Url::parse(&new_variant).unwrap().path_segments().unwrap().nth(3).unwrap().to_owned();
|
||||
assert_eq!(previous_session, current_session, "the shared origin session stays alive");
|
||||
// Reentry releases the user slot before the origin retry finishes. The
|
||||
// pending lease may still wait for that retry's publishable media.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
let playlist = loop {
|
||||
let response = new_player.get(&new_variant).send().await.unwrap();
|
||||
if response.status() == 200 {
|
||||
break response;
|
||||
}
|
||||
assert_eq!(response.status(), 503, "new live playlist rejected: {}", fixture.log());
|
||||
assert!(tokio::time::Instant::now() < deadline, "new live playlist never recovered: {}", fixture.log());
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
};
|
||||
let new_segment = resolve_media(&new_variant, &playlist.text().await.unwrap(), 1);
|
||||
let repaired = new_player.get(new_segment).send().await.unwrap();
|
||||
assert_eq!(repaired.status(), 200, "new player cannot use the successful revision: {}", fixture.log());
|
||||
assert_eq!(repaired.bytes().await.unwrap().as_ref(), BODY);
|
||||
assert_eq!(client.get(&segment).send().await.unwrap().status(), 404);
|
||||
fixture.process.kill().await.unwrap();
|
||||
}
|
||||
@@ -26,6 +26,8 @@ pub struct HlsCtx {
|
||||
}
|
||||
|
||||
impl HlsCtx {
|
||||
/// Give request-independent lease completion its authoritative user registry.
|
||||
pub fn attach_lease_user_accounting(&self) { self.hls_proxy.bind_lease_user_accounting(&self.active_users); }
|
||||
/// The subset admission reads: it decides over connections and users, not
|
||||
/// over the cache.
|
||||
pub fn admission_ctx(&self) -> tuliprox_session::admission::AdmissionCtx {
|
||||
@@ -38,6 +40,7 @@ impl HlsCtx {
|
||||
|
||||
/// A weak handle, for state the proxy itself owns.
|
||||
pub fn downgrade(&self) -> WeakHlsCtx {
|
||||
self.attach_lease_user_accounting();
|
||||
WeakHlsCtx {
|
||||
app_config: Arc::clone(&self.app_config),
|
||||
hls_proxy: Arc::downgrade(&self.hls_proxy),
|
||||
@@ -128,12 +131,14 @@ impl HlsCtx {
|
||||
None,
|
||||
));
|
||||
|
||||
Self {
|
||||
let ctx = Self {
|
||||
hls_proxy: Arc::new(crate::manager::HlsProxyManager::new()),
|
||||
app_config: Arc::clone(&app_config),
|
||||
active_provider,
|
||||
connection_manager,
|
||||
active_users,
|
||||
}
|
||||
};
|
||||
ctx.attach_lease_user_accounting();
|
||||
ctx
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,11 +327,13 @@ mod identity;
|
||||
mod lifecycle;
|
||||
mod publication;
|
||||
mod revision_bindings;
|
||||
mod revision_recovery;
|
||||
mod runtime_policy;
|
||||
mod terminal;
|
||||
mod timing;
|
||||
|
||||
pub use self::identity::new_hls_access_lease_id;
|
||||
pub(crate) use self::revision_recovery::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome};
|
||||
use self::{
|
||||
error::HlsAvailabilityEvidenceAdvanceError,
|
||||
identity::HlsMediaLeasePlaybackIdentity,
|
||||
|
||||
@@ -26,6 +26,12 @@ pub(super) enum HlsMediaLeasePlaybackIdentity {
|
||||
|
||||
impl HlsLeaseManifestPublicationGuard {
|
||||
pub(crate) const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms }
|
||||
pub(crate) const fn media_identity(self) -> HlsMediaLeaseIdentity {
|
||||
HlsMediaLeaseIdentity {
|
||||
issued_at_ms: self.issued_at_ms,
|
||||
playback: HlsMediaLeasePlaybackIdentity::Live { admission_generation: self.admission_generation },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsAccessLease {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::{
|
||||
HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode,
|
||||
ProxySessionId,
|
||||
lease_state_allows_use, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot,
|
||||
HlsLeasePlaybackMode, ProxySessionId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -50,14 +50,19 @@ impl HlsAccessLeaseStore {
|
||||
) -> Option<super::super::SegmentRevisionGuard> {
|
||||
self.by_lease_id.get(lease_id)?.revision_bindings.as_ref()?.revisions.get(&seq).cloned()
|
||||
}
|
||||
/// Reserves the lease's single startup opportunity for its first authorized
|
||||
/// full-segment request, including a cache hit. The caller excludes partial
|
||||
/// ranges and holds this lock across lease-identity validation and lookup.
|
||||
/// Bindings include still-retained URIs from earlier published windows.
|
||||
pub fn try_claim_progressive_startup(&mut self, lease_id: &HlsAccessLeaseId, seq: u64, now_ms: u64) -> bool {
|
||||
let Some(lease) = self.by_lease_id.get_mut(lease_id) else {
|
||||
return false;
|
||||
};
|
||||
if lease.playback_mode != HlsLeasePlaybackMode::Live
|
||||
if !lease_state_allows_use(lease.state)
|
||||
|| lease.playback_mode != HlsLeasePlaybackMode::Live
|
||||
|| lease.progressive_startup_claimed
|
||||
|| lease.validity_due_at_ms() <= now_ms
|
||||
|| lease.last_manifest_snapshot.as_ref().is_none_or(|snapshot| snapshot.first_proxy_seq != seq)
|
||||
|| lease.revision_bindings.as_ref().is_none_or(|bindings| !bindings.revisions.contains_key(&seq))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -74,7 +79,9 @@ pub(super) fn commit_lease_revision_bindings(lease: &mut HlsAccessLease, snapsho
|
||||
|| current.is_some_and(|current| {
|
||||
current.retired_through.is_some_and(|retired| *seq <= retired)
|
||||
|| current.revisions.get(seq).is_some_and(|bound| {
|
||||
bound != revision && !bound.revision().is_replaceable_in(publication.mode)
|
||||
bound != revision
|
||||
&& (!bound.revision().is_replaceable_in(publication.mode)
|
||||
|| revision.revision().id() <= bound.revision().id())
|
||||
})
|
||||
})
|
||||
}) {
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! One transaction for repairing an already published segment binding.
|
||||
use super::{
|
||||
HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsAccessLeaseState, HlsAccessLeaseStore, HlsLeasePlaybackMode,
|
||||
HlsMediaLeaseIdentity,
|
||||
};
|
||||
use crate::{HlsSession, SegmentCacheStatus, SegmentRevisionGuard, SegmentRevisionState};
|
||||
use std::sync::{atomic::Ordering, Arc};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct HlsRevisionRecoveryGuard {
|
||||
identity: HlsMediaLeaseIdentity,
|
||||
snapshot_generation: u64,
|
||||
pub(crate) revision: SegmentRevisionGuard,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HlsRevisionRecoveryOutcome {
|
||||
Unchanged,
|
||||
ReplacementBound,
|
||||
RetryPending,
|
||||
ReentryRequired,
|
||||
}
|
||||
|
||||
impl HlsRevisionRecoveryGuard {
|
||||
pub(crate) fn completion_identity(&self) -> (u64, u64) {
|
||||
(self.identity.lease_issued_at_ms(), self.snapshot_generation)
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsAccessLeaseStore {
|
||||
pub(crate) fn revision_recovery_guard(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
seq: u64,
|
||||
) -> Option<HlsRevisionRecoveryGuard> {
|
||||
let lease = self.by_lease_id.get(lease_id)?;
|
||||
Some(HlsRevisionRecoveryGuard {
|
||||
identity: lease.media_identity()?,
|
||||
snapshot_generation: lease.manifest_snapshot_generation,
|
||||
revision: self.published_segment_revision(lease_id, seq)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn revision_recovery_is_current(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
expected.identity.is_live()
|
||||
&& !session.is_gc_marked_for_removal()
|
||||
&& self.media_identity_is_current(lease_id, &session.proxy_session_id, expected.identity, now_ms)
|
||||
&& self.by_lease_id.get(lease_id).is_some_and(|lease| {
|
||||
lease.manifest_snapshot_generation == expected.snapshot_generation
|
||||
&& self.published_segment_revision(lease_id, expected.revision.revision().key.proxy_seq).as_ref()
|
||||
== Some(&expected.revision)
|
||||
})
|
||||
}
|
||||
|
||||
/// Caller holds the lease store and session locks in that order. The guard
|
||||
/// fences the exact lease incarnation, snapshot and old revision together.
|
||||
pub(crate) fn recover_revision_binding(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> HlsRevisionRecoveryOutcome {
|
||||
use HlsRevisionRecoveryOutcome::{ReentryRequired, ReplacementBound, RetryPending, Unchanged};
|
||||
if !self.revision_recovery_is_current(lease_id, expected, session, now_ms)
|
||||
|| !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed)
|
||||
{
|
||||
return Unchanged;
|
||||
}
|
||||
let old = expected.revision.revision();
|
||||
let seq = old.key.proxy_seq;
|
||||
if old.was_exposed() {
|
||||
return ReentryRequired;
|
||||
}
|
||||
if !old.origin_binding_is_current(session) {
|
||||
return ReentryRequired;
|
||||
}
|
||||
let mode = self.by_lease_id.get(lease_id).and_then(|lease| {
|
||||
lease.last_manifest_snapshot.as_ref()?.startup_revisions.as_ref().map(|publication| publication.mode)
|
||||
});
|
||||
if !mode.is_some_and(|mode| old.is_replaceable_in(mode)) {
|
||||
return ReentryRequired;
|
||||
}
|
||||
let candidate = session.startup.as_ref().and_then(|startup| startup.revisions.get(&seq));
|
||||
let replacement = candidate.filter(|candidate| {
|
||||
*candidate != &expected.revision
|
||||
&& candidate.revision().key.proxy_seq == seq
|
||||
&& candidate.revision().id() > old.id()
|
||||
&& candidate.revision().key.kind == old.key.kind
|
||||
&& candidate.revision().origin_binding_is_current(session)
|
||||
&& candidate.revision().is_complete()
|
||||
&& session
|
||||
.segments
|
||||
.get(&seq)
|
||||
.is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. }))
|
||||
});
|
||||
let Some(replacement) = replacement else {
|
||||
return if session.segments.get(&seq).is_some_and(|entry| {
|
||||
matches!(
|
||||
entry.status,
|
||||
SegmentCacheStatus::Discovered
|
||||
| SegmentCacheStatus::Queued { .. }
|
||||
| SegmentCacheStatus::Fetching { .. }
|
||||
| SegmentCacheStatus::FailedRetryable { .. }
|
||||
| SegmentCacheStatus::CapacityDeferred { .. }
|
||||
)
|
||||
}) {
|
||||
RetryPending
|
||||
} else {
|
||||
ReentryRequired
|
||||
};
|
||||
};
|
||||
if self.availability_evidence_can_advance().is_err() {
|
||||
return RetryPending;
|
||||
}
|
||||
let Some(lease) = self.by_lease_id.get_mut(lease_id) else { return Unchanged };
|
||||
let Some(generation) = lease.manifest_snapshot_generation.checked_add(1) else { return RetryPending };
|
||||
let Some(bindings) = lease.revision_bindings.as_mut() else { return Unchanged };
|
||||
Arc::make_mut(bindings).revisions.insert(seq, replacement.clone());
|
||||
lease.manifest_snapshot_generation = generation;
|
||||
if let Some(snapshot) = lease.last_manifest_snapshot.as_mut() {
|
||||
snapshot.snapshot_generation = generation;
|
||||
if let Some(publication) = snapshot.startup_revisions.as_mut() {
|
||||
if Arc::make_mut(publication).revisions.contains_key(&seq) {
|
||||
Arc::make_mut(publication).revisions.insert(seq, replacement.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = self.advance_availability_evidence(&session.proxy_session_id);
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={seq} generation={generation} old={:?} new={:?} prefix={} exposed={} outcome=replacement_bound",
|
||||
crate::safe_hls_access_lease_id(lease_id), old.id(), replacement.revision().id(),
|
||||
old.prefix_available.load(Ordering::Acquire), old.was_exposed(),
|
||||
);
|
||||
ReplacementBound
|
||||
}
|
||||
|
||||
/// An identity-checked, idempotent live -> ended transition. Removal and all
|
||||
/// existing external cleanup happen through the manager's prepared removal.
|
||||
pub(crate) fn end_revision_playback_if_current(
|
||||
&mut self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
expected: &HlsRevisionRecoveryGuard,
|
||||
session: &HlsSession,
|
||||
now_ms: u64,
|
||||
) -> Option<HlsAccessLeaseRemovalPreparation> {
|
||||
if !self.revision_recovery_is_current(lease_id, expected, session, now_ms)
|
||||
|| !matches!(expected.revision.revision().state(), SegmentRevisionState::Failed)
|
||||
|| self.availability_evidence_can_advance().is_err()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let lease = self.by_lease_id.get_mut(lease_id)?;
|
||||
if lease.playback_mode != HlsLeasePlaybackMode::Live {
|
||||
return None;
|
||||
}
|
||||
lease.end_playback();
|
||||
lease.state = HlsAccessLeaseState::Denied;
|
||||
let _ = self.advance_availability_evidence(&session.proxy_session_id);
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=reentry_required",
|
||||
crate::safe_hls_access_lease_id(lease_id), expected.revision.revision().key.proxy_seq,
|
||||
expected.snapshot_generation, expected.revision.revision().id(),
|
||||
expected.revision.revision().prefix_available.load(Ordering::Acquire), expected.revision.revision().was_exposed(),
|
||||
);
|
||||
self.prepare_access_lease_removal(lease_id)
|
||||
}
|
||||
|
||||
pub(crate) fn revision_bound_lease_ids(&self, session: &crate::ProxySessionId, seq: u64) -> Vec<HlsAccessLeaseId> {
|
||||
self.by_lease_id
|
||||
.values()
|
||||
.filter(|lease| {
|
||||
lease.proxy_session_id == *session
|
||||
&& lease.playback_mode == HlsLeasePlaybackMode::Live
|
||||
&& lease.revision_bindings.as_ref().is_some_and(|bindings| bindings.revisions.contains_key(&seq))
|
||||
})
|
||||
.map(|lease| lease.lease_id.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -66,20 +66,110 @@ fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io::
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_only_accepts_the_published_head_once() {
|
||||
fn startup_claim_fixture() -> std::io::Result<(HlsAccessLeaseStore, HlsAccessLeaseId)> {
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let lease_id = HlsAccessLeaseId("claim-lease".into());
|
||||
let mut leases = HlsAccessLeaseStore::default();
|
||||
leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(10), 2000).is_committed());
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
if let Some(lease) = leases.by_lease_id.get_mut(&lease_id) {
|
||||
lease.playback_cursor.first_requested_proxy_seq = Some(41);
|
||||
let store = crate::SegmentRevisionStore::default();
|
||||
let mut revisions = std::collections::BTreeMap::new();
|
||||
for seq in [40, 41] {
|
||||
let revision = store.create(session.clone(), seq, crate::SegmentRevisionKind::Raw)?;
|
||||
revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
revisions.insert(seq, revision);
|
||||
}
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2101));
|
||||
let mut snapshot = manifest_snapshot(10);
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: 40,
|
||||
revisions,
|
||||
}));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2000).is_committed());
|
||||
Ok((leases, lease_id))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_accepts_the_first_requested_published_segment_once() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 42, 2100), "unpublished URI cannot consume startup");
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101), "the player may start after the head");
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2102));
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2102));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_accepts_a_retained_uri_after_the_playlist_head_moves() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let mut snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
snapshot.first_proxy_seq = 41;
|
||||
snapshot.visible_segments = Arc::from([snapshot.visible_segments[1].clone()]);
|
||||
Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).revisions.remove(&40);
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed());
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2201), "an earlier published URI is still bound");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_lease_gets_its_own_startup_claim_for_the_shared_revision() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
let other_id = HlsAccessLeaseId("other-lease".into());
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
assert!(leases.prepare_access_lease(lease(other_id.clone(), &session.0, 2000)));
|
||||
assert!(publish_manifest_snapshot(&mut leases, &other_id, &session, snapshot, 2100).is_committed());
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2101));
|
||||
assert!(leases.try_claim_progressive_startup(&other_id, 41, 2102));
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2103));
|
||||
assert!(!leases.try_claim_progressive_startup(&other_id, 40, 2103));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_cannot_be_revived_by_a_playlist_refresh_or_cursor_seek() -> std::io::Result<()> {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
let session = ProxySessionId("proxy".into());
|
||||
let snapshot = leases.by_lease_id[&lease_id].last_manifest_snapshot.clone().expect("snapshot");
|
||||
assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, snapshot, 2200).is_committed());
|
||||
let _ = leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_cursor.record_request_started(100, 2201);
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2202));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_claim_is_scoped_to_a_valid_live_lease() -> std::io::Result<()> {
|
||||
for state in [super::HlsAccessLeaseState::Denied, super::HlsAccessLeaseState::Expired] {
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
leases.by_lease_id.get_mut(&lease_id).expect("lease").state = state;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
}
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 16_000));
|
||||
let (mut leases, lease_id) = startup_claim_fixture()?;
|
||||
leases.by_lease_id.get_mut(&lease_id).expect("lease").playback_mode = super::HlsLeasePlaybackMode::Ended;
|
||||
assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_startup_claims_allow_exactly_one_request() -> std::io::Result<()> {
|
||||
let (leases, lease_id) = startup_claim_fixture()?;
|
||||
let leases = Arc::new(tokio::sync::RwLock::new(leases));
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let claim = |seq| {
|
||||
let leases = Arc::clone(&leases);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
let lease_id = lease_id.clone();
|
||||
tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
leases.write().await.try_claim_progressive_startup(&lease_id, seq, 2100)
|
||||
})
|
||||
};
|
||||
let (first, second) = tokio::join!(claim(40), claim(41));
|
||||
assert_ne!(first.expect("first task"), second.expect("second task"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -77,6 +77,7 @@ pub struct HlsProxyManager {
|
||||
transient_resources: Arc<TransientResourceStore>,
|
||||
access_leases: Arc<RwLock<HlsAccessLeaseStore>>,
|
||||
lifecycle: Arc<HlsLifecycleManager>,
|
||||
lease_completion: Arc<lease_completion::HlsLeaseCompletion>,
|
||||
account_overlap_cooldowns: Arc<RwLock<HashMap<HlsAccountOverlapCooldownKey, HlsAccountOverlapCooldown>>>,
|
||||
metrics: Arc<HlsCacheMetrics>,
|
||||
qos: Arc<HlsQosRegistry>,
|
||||
@@ -153,8 +154,10 @@ mod account_overlap;
|
||||
mod activity;
|
||||
mod cleanup;
|
||||
mod config;
|
||||
mod lease_completion;
|
||||
mod lifecycle;
|
||||
mod publication;
|
||||
mod revision_recovery;
|
||||
mod terminal;
|
||||
|
||||
pub use self::lifecycle::exec_hls_lifecycle;
|
||||
@@ -162,6 +165,7 @@ pub use self::lifecycle::exec_hls_lifecycle;
|
||||
use self::lifecycle::hls_session_idle_protection_retry_at;
|
||||
#[cfg(test)]
|
||||
use self::publication::HlsRecoveryExecutionState;
|
||||
pub(crate) use self::revision_recovery::HlsLeaseRuntime;
|
||||
use self::{
|
||||
account_overlap::{HlsAccountOverlapCooldown, HlsAccountOverlapCooldownKey},
|
||||
activity::hls_key_readiness_evidence_is_current,
|
||||
|
||||
@@ -111,7 +111,7 @@ impl HlsProxyManager {
|
||||
gc.install_revision_store(&revision_store);
|
||||
segment_cache.install_revision_metrics(&metrics);
|
||||
let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default());
|
||||
Self {
|
||||
let manager = Self {
|
||||
sessions,
|
||||
segment_cache,
|
||||
segment_repair,
|
||||
@@ -135,6 +135,7 @@ impl HlsProxyManager {
|
||||
transient_resources: Arc::new(TransientResourceStore::new()),
|
||||
access_leases,
|
||||
lifecycle,
|
||||
lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()),
|
||||
account_overlap_cooldowns,
|
||||
metrics,
|
||||
qos,
|
||||
@@ -146,7 +147,9 @@ impl HlsProxyManager {
|
||||
availability_reevaluations,
|
||||
terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()),
|
||||
startup_observability: Arc::new(HlsStartupObservability::default()),
|
||||
}
|
||||
};
|
||||
manager.install_worker_revision_recovery();
|
||||
manager
|
||||
}
|
||||
|
||||
pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() }
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
//! Lease completion belongs to the runtime, not to the HTTP future waiting for it.
|
||||
use super::{HlsAccessLeaseId, HlsAccessLeaseRemovalPreparation, HlsProxyManager, ProxySessionId};
|
||||
use crate::{safe_proxy_session_id, sync_ext::MutexExt, HlsTerminalTailProtectionRemoval};
|
||||
use arc_swap::ArcSwapOption;
|
||||
use log::debug;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
future::Future,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use tokio::sync::{watch, Notify};
|
||||
use tuliprox_session::ActiveUserManager;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub(super) struct LeaseCompletionKey {
|
||||
pub(super) lease_id: HlsAccessLeaseId,
|
||||
pub(super) session_id: ProxySessionId,
|
||||
pub(super) issued_at_ms: u64,
|
||||
pub(super) snapshot_generation: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct HlsLeaseCompletion {
|
||||
users: ArcSwapOption<ActiveUserManager>,
|
||||
users_bound: Notify,
|
||||
jobs: Mutex<HashMap<LeaseCompletionKey, watch::Receiver<Option<bool>>>>,
|
||||
}
|
||||
|
||||
impl HlsLeaseCompletion {
|
||||
fn bind_users(&self, users: &Arc<ActiveUserManager>) {
|
||||
self.users.store(Some(Arc::clone(users)));
|
||||
self.users_bound.notify_waiters();
|
||||
}
|
||||
|
||||
pub(super) async fn users(&self) -> Arc<ActiveUserManager> {
|
||||
loop {
|
||||
let notified = self.users_bound.notified();
|
||||
if let Some(users) = self.users.load_full() {
|
||||
return users;
|
||||
}
|
||||
// Before runtime attachment the lease remains its cleanup ticket;
|
||||
// an absent accounting owner must never silently skip the release.
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run(
|
||||
self: &Arc<Self>,
|
||||
key: LeaseCompletionKey,
|
||||
completion: impl Future<Output = bool> + Send + 'static,
|
||||
) -> bool {
|
||||
let mut receiver = {
|
||||
let mut jobs = self.jobs.lock_unpoisoned();
|
||||
if let Some(receiver) = jobs.get(&key) {
|
||||
receiver.clone()
|
||||
} else {
|
||||
let (sender, receiver) = watch::channel(None);
|
||||
jobs.insert(key.clone(), receiver.clone());
|
||||
let owner = Arc::clone(self);
|
||||
// Registered and spawned before the first await. Cancelling any
|
||||
// waiter cannot cancel or erase the runtime's completion job.
|
||||
tokio::spawn(async move {
|
||||
let result = completion.await;
|
||||
owner.jobs.lock_unpoisoned().remove(&key);
|
||||
let _ = sender.send(Some(result));
|
||||
});
|
||||
receiver
|
||||
}
|
||||
};
|
||||
loop {
|
||||
if let Some(result) = *receiver.borrow_and_update() {
|
||||
return result;
|
||||
}
|
||||
if receiver.changed().await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pending_count(&self) -> usize { self.jobs.lock_unpoisoned().len() }
|
||||
}
|
||||
|
||||
impl HlsProxyManager {
|
||||
/// Attach the server's authoritative user registry before serving leases.
|
||||
pub fn bind_lease_user_accounting(&self, users: &Arc<ActiveUserManager>) {
|
||||
self.lease_completion.bind_users(users);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pending_lease_completions_for_test(&self) -> usize { self.lease_completion.pending_count() }
|
||||
}
|
||||
|
||||
impl super::HlsLeaseRuntime {
|
||||
pub(super) async fn remove_prepared_access_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
let key = LeaseCompletionKey {
|
||||
lease_id: lease_id.clone(),
|
||||
session_id: preparation.proxy_session_id.clone(),
|
||||
issued_at_ms: preparation.issued_at_ms,
|
||||
snapshot_generation: None,
|
||||
};
|
||||
let runtime = self.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
let preparation = preparation.clone();
|
||||
self.lease_completion
|
||||
.run(key, async move { runtime.complete_prepared_lease(&lease_id, &preparation).await })
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl super::HlsLeaseRuntime {
|
||||
pub(super) async fn complete_prepared_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &crate::lease::HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
// Hold the index and lease-store fences through external cleanup. A
|
||||
// replacement with the same ID must not acquire QoS/repair ownership
|
||||
// between removing the old incarnation and clearing those registries.
|
||||
// Reject stale tickets before waiting for runtime attachment.
|
||||
if !self.access_leases.read().await.prepare_access_lease_removal(lease_id).is_some_and(|current| {
|
||||
current.proxy_session_id == preparation.proxy_session_id && current.issued_at_ms == preparation.issued_at_ms
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
let users = self.lease_completion.users().await;
|
||||
let current = self.sessions.hold_current_proxy_session(&preparation.proxy_session_id).await;
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let Some(lease) = leases.response_snapshot(
|
||||
lease_id,
|
||||
&preparation.proxy_session_id,
|
||||
tuliprox_core::utils::current_time_millis(),
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
if lease.issued_at_ms != preparation.issued_at_ms {
|
||||
return false;
|
||||
}
|
||||
users.release_session_streams_and_counted_reservation(&lease.username, &lease.user_session_token).await;
|
||||
self.terminal_pending.cancel_lease(lease_id);
|
||||
self.terminal_commit_retries.cancel_lease(lease_id);
|
||||
if let Some(current) = current.as_ref() {
|
||||
let session = current.session();
|
||||
let mut session = session.write().await;
|
||||
let finalized_manifest_released =
|
||||
session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms);
|
||||
if finalized_manifest_released {
|
||||
debug!(
|
||||
"HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
if let Some(generation) = preparation.terminal_protection_generation {
|
||||
let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation);
|
||||
match removal {
|
||||
HlsTerminalTailProtectionRemoval::Removed => {
|
||||
debug!(
|
||||
"HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0
|
||||
);
|
||||
}
|
||||
HlsTerminalTailProtectionRemoval::Missing => {}
|
||||
HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => {
|
||||
debug!(
|
||||
"HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0,
|
||||
actual.0
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if session.remove_terminal_tail_protection(lease_id).is_some() {
|
||||
debug!(
|
||||
"HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
self.standalone_custom_access.remove(lease_id);
|
||||
self.segment_repair.remove_access_lease_window(lease_id).await;
|
||||
self.startup_observability.remove_access_lease(lease_id);
|
||||
self.qos.remove_access_lease(lease_id).await;
|
||||
// Keep the exact lease present until all release steps have finished.
|
||||
leases.remove_access_lease_if_preparation_matches(lease_id, preparation).is_some()
|
||||
}
|
||||
}
|
||||
@@ -51,8 +51,6 @@ impl HlsProxyManager {
|
||||
}
|
||||
|
||||
pub(super) async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) {
|
||||
self.terminal_pending.cancel_lease(lease_id);
|
||||
self.terminal_commit_retries.cancel_lease(lease_id);
|
||||
let Some(preparation) = self.access_leases.read().await.prepare_access_lease_removal(lease_id) else {
|
||||
return;
|
||||
};
|
||||
@@ -65,65 +63,15 @@ impl HlsProxyManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes one exact lease incarnation before releasing any media
|
||||
/// protection owned by it. A stale preparation is intentionally
|
||||
/// non-destructive because the replacement lease may reuse the same ID.
|
||||
/// Complete one exact lease incarnation in the shared, request-independent
|
||||
/// runtime. The lease is retired after accounting and media protection are
|
||||
/// released; stale preparations cannot affect a replacement with the same ID.
|
||||
pub(super) async fn remove_prepared_access_lease(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
preparation: &HlsAccessLeaseRemovalPreparation,
|
||||
) -> bool {
|
||||
let removed =
|
||||
self.access_leases.write().await.remove_access_lease_if_preparation_matches(lease_id, preparation);
|
||||
if removed.is_none() {
|
||||
return false;
|
||||
}
|
||||
if let Some(session) = self.sessions.get_by_proxy_session_id(&preparation.proxy_session_id).await {
|
||||
let mut session = session.write().await;
|
||||
let finalized_manifest_released =
|
||||
session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms);
|
||||
if finalized_manifest_released {
|
||||
debug!(
|
||||
"HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
if let Some(generation) = preparation.terminal_protection_generation {
|
||||
let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation);
|
||||
match removal {
|
||||
HlsTerminalTailProtectionRemoval::Removed => {
|
||||
debug!(
|
||||
"HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0
|
||||
);
|
||||
}
|
||||
HlsTerminalTailProtectionRemoval::Missing => {}
|
||||
HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => {
|
||||
debug!(
|
||||
"HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id),
|
||||
generation.0,
|
||||
actual.0
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if session.remove_terminal_tail_protection(lease_id).is_some() {
|
||||
debug!(
|
||||
"HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed",
|
||||
super::super::safe_hls_access_lease_id(lease_id),
|
||||
safe_proxy_session_id(&preparation.proxy_session_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
self.standalone_custom_access.remove(lease_id);
|
||||
self.segment_repair.remove_access_lease_window(lease_id).await;
|
||||
self.startup_observability.remove_access_lease(lease_id);
|
||||
self.qos.remove_access_lease(lease_id).await;
|
||||
true
|
||||
self.revision_recovery_runtime().remove_prepared_access_lease(lease_id, preparation).await
|
||||
}
|
||||
|
||||
pub(super) async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) {
|
||||
@@ -205,6 +153,7 @@ impl HlsProxyManager {
|
||||
event: HlsLifecycleEvent,
|
||||
now_ms: u64,
|
||||
) {
|
||||
self.bind_lease_user_accounting(active_users);
|
||||
if !self.is_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -214,7 +163,8 @@ impl HlsProxyManager {
|
||||
let mut should_sync_session = false;
|
||||
if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await {
|
||||
should_sync_session = true;
|
||||
if let Some(release) = &snapshot.idle_release {
|
||||
let ending = matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied);
|
||||
if let Some(release) = snapshot.idle_release.as_ref().filter(|_| !ending) {
|
||||
active_users
|
||||
.release_session_streams_and_counted_reservation(
|
||||
&release.username,
|
||||
@@ -228,7 +178,7 @@ impl HlsProxyManager {
|
||||
super::super::safe_user_session_token(&release.user_session_token)
|
||||
);
|
||||
}
|
||||
if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) {
|
||||
if ending {
|
||||
self.remove_access_lease(&snapshot.lease_id).await;
|
||||
debug!(
|
||||
"HLS access lease removed: lease={} proxy_session={} state={}",
|
||||
@@ -385,6 +335,7 @@ impl HlsProxyManager {
|
||||
}
|
||||
|
||||
pub fn exec_hls_lifecycle(ctx: &HlsCtx, cancel_token: &CancellationToken) {
|
||||
ctx.attach_lease_user_accounting();
|
||||
let hls_proxy = Arc::clone(&ctx.hls_proxy);
|
||||
let active_users = Arc::clone(&ctx.active_users);
|
||||
let active_provider = Arc::clone(&ctx.active_provider);
|
||||
|
||||
@@ -160,7 +160,7 @@ impl HlsProxyManager {
|
||||
gc.install_revision_store(&revision_store);
|
||||
segment_cache.install_revision_metrics(&metrics);
|
||||
let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default());
|
||||
Self {
|
||||
let manager = Self {
|
||||
sessions,
|
||||
segment_cache,
|
||||
segment_repair,
|
||||
@@ -184,6 +184,7 @@ impl HlsProxyManager {
|
||||
transient_resources: Arc::new(TransientResourceStore::new()),
|
||||
access_leases,
|
||||
lifecycle,
|
||||
lease_completion: Arc::new(super::lease_completion::HlsLeaseCompletion::default()),
|
||||
account_overlap_cooldowns,
|
||||
metrics,
|
||||
qos,
|
||||
@@ -195,7 +196,9 @@ impl HlsProxyManager {
|
||||
availability_reevaluations,
|
||||
terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()),
|
||||
startup_observability: Arc::new(HlsStartupObservability::default()),
|
||||
}
|
||||
};
|
||||
manager.install_worker_revision_recovery();
|
||||
manager
|
||||
}
|
||||
|
||||
pub fn sessions(&self) -> &Arc<HlsSessionStore> { &self.sessions }
|
||||
@@ -475,6 +478,18 @@ impl HlsProxyManager {
|
||||
published_transient_resource_ids: HlsPublishedTransientResourceIds,
|
||||
now_ms: u64,
|
||||
) -> HlsLeaseManifestPublicationOutcome {
|
||||
if snapshot.startup_revisions.is_some() {
|
||||
return self
|
||||
.commit_recovered_startup_manifest(
|
||||
lease_id,
|
||||
proxy_session_id,
|
||||
expected,
|
||||
snapshot,
|
||||
published_transient_resource_ids,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation;
|
||||
let lease_issued_at_ms = expected.lease_issued_at_ms();
|
||||
let Some(finalized_manifest_generation) = finalized_manifest_generation else {
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
use crate::{
|
||||
lease::{HlsRevisionRecoveryGuard, HlsRevisionRecoveryOutcome},
|
||||
HlsAccessLeaseId, HlsMediaLeaseIdentity, HlsProxyManager, ProxySessionId,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Manager-owned resources used by fetch completion, even when an origin I/O
|
||||
/// context is absent. Sharing this handle avoids a second lease cleanup path.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct HlsLeaseRuntime {
|
||||
pub(super) sessions: Arc<crate::HlsSessionStore>,
|
||||
pub(super) lease_completion: Arc<super::lease_completion::HlsLeaseCompletion>,
|
||||
pub(super) access_leases: Arc<tokio::sync::RwLock<crate::HlsAccessLeaseStore>>,
|
||||
pub(super) standalone_custom_access: Arc<crate::runtime_custom_tail::HlsStandaloneCustomAccessStore>,
|
||||
pub(super) segment_repair: Arc<crate::HlsSegmentRepairManager>,
|
||||
pub(super) startup_observability: Arc<crate::HlsStartupObservability>,
|
||||
pub(super) qos: Arc<crate::HlsQosRegistry>,
|
||||
pub(super) terminal_pending: Arc<crate::terminal_pending::HlsTerminalPendingCoordinator>,
|
||||
pub(super) terminal_commit_retries: Arc<crate::terminal_commit::HlsTerminalCommitRetryCoordinator>,
|
||||
}
|
||||
|
||||
impl HlsProxyManager {
|
||||
pub(super) fn revision_recovery_runtime(&self) -> HlsLeaseRuntime {
|
||||
HlsLeaseRuntime {
|
||||
sessions: Arc::clone(&self.sessions),
|
||||
lease_completion: Arc::clone(&self.lease_completion),
|
||||
access_leases: Arc::clone(&self.access_leases),
|
||||
standalone_custom_access: Arc::clone(&self.standalone_custom_access),
|
||||
segment_repair: Arc::clone(&self.segment_repair),
|
||||
startup_observability: Arc::clone(&self.startup_observability),
|
||||
qos: Arc::clone(&self.qos),
|
||||
terminal_pending: Arc::clone(&self.terminal_pending),
|
||||
terminal_commit_retries: Arc::clone(&self.terminal_commit_retries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn install_worker_revision_recovery(&self) {
|
||||
let _ = self.segment_worker_pool.revision_recovery.set(Arc::new(self.revision_recovery_runtime()));
|
||||
}
|
||||
|
||||
pub(super) async fn commit_recovered_startup_manifest(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
expected: crate::lease::HlsLeaseManifestPublicationGuard,
|
||||
mut snapshot: crate::media_reserve::HlsLeaseManifestSnapshot,
|
||||
resources: crate::HlsPublishedTransientResourceIds,
|
||||
now_ms: u64,
|
||||
) -> crate::lease::HlsLeaseManifestPublicationOutcome {
|
||||
use crate::lease::{
|
||||
HlsLeaseManifestPublicationOutcome as Outcome, HlsLeaseManifestPublicationRejectReason as Reject,
|
||||
};
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else {
|
||||
return Outcome::Rejected(Reject::SessionMismatch);
|
||||
};
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
if !leases.media_identity_is_current(lease_id, session_id, expected.media_identity(), now_ms)
|
||||
|| session.is_gc_marked_for_removal()
|
||||
{
|
||||
return Outcome::Rejected(Reject::LeaseUnavailable);
|
||||
}
|
||||
if let Some(publication) = snapshot.startup_revisions.as_mut() {
|
||||
for (seq, revision) in &mut std::sync::Arc::make_mut(publication).revisions {
|
||||
if let Some(guard) = leases.revision_recovery_guard(lease_id, *seq) {
|
||||
match leases.recover_revision_binding(lease_id, &guard, &session, now_ms) {
|
||||
HlsRevisionRecoveryOutcome::ReentryRequired => {
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await;
|
||||
return Outcome::Rejected(Reject::LeaseUnavailable);
|
||||
}
|
||||
HlsRevisionRecoveryOutcome::RetryPending => return Outcome::Rejected(Reject::RevisionConflict),
|
||||
HlsRevisionRecoveryOutcome::Unchanged | HlsRevisionRecoveryOutcome::ReplacementBound => {}
|
||||
}
|
||||
if let Some(bound) = leases.published_segment_revision(lease_id, *seq) {
|
||||
if revision.revision().id() < bound.revision().id()
|
||||
&& matches!(revision.revision().state(), crate::SegmentRevisionState::Failed)
|
||||
{
|
||||
*revision = bound;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Publication cannot introduce a revision left by a retired
|
||||
// fetch, nor use a caller-supplied candidate outside startup.
|
||||
if session.startup.as_ref().and_then(|startup| startup.revisions.get(seq)) != Some(revision)
|
||||
&& leases.published_segment_revision(lease_id, *seq).as_ref() != Some(revision)
|
||||
{
|
||||
return Outcome::Rejected(Reject::RevisionConflict);
|
||||
}
|
||||
}
|
||||
}
|
||||
leases.commit_manifest_publication_with_resources(lease_id, session_id, expected, snapshot, resources, now_ms)
|
||||
}
|
||||
|
||||
/// Resolves one published URI against the exact current session and lease.
|
||||
/// The index -> lease store -> session order matches publication/cutover.
|
||||
pub(crate) async fn recover_access_lease_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
identity: HlsMediaLeaseIdentity,
|
||||
seq: u64,
|
||||
now_ms: u64,
|
||||
) -> HlsRevisionRecoveryOutcome {
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
};
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
if !leases.media_identity_is_current(lease_id, session_id, identity, now_ms) {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
}
|
||||
let Some(guard) = leases.revision_recovery_guard(lease_id, seq) else {
|
||||
return HlsRevisionRecoveryOutcome::Unchanged;
|
||||
};
|
||||
let outcome = leases.recover_revision_binding(lease_id, &guard, &session, now_ms);
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
if outcome == HlsRevisionRecoveryOutcome::ReentryRequired {
|
||||
self.end_access_lease_for_failed_revision(lease_id, session_id, &guard, now_ms).await;
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
pub(crate) async fn end_access_lease_for_failed_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
self.revision_recovery_runtime().end_access_lease_for_failed_revision(lease_id, session_id, guard, now_ms).await
|
||||
}
|
||||
}
|
||||
|
||||
impl HlsLeaseRuntime {
|
||||
/// Retire only the observed binding/snapshot/lease incarnation. Repeated
|
||||
/// callers and completions from older snapshots are intentionally harmless.
|
||||
pub(crate) async fn end_access_lease_for_failed_revision(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
let (issued_at_ms, snapshot_generation) = guard.completion_identity();
|
||||
let key = super::lease_completion::LeaseCompletionKey {
|
||||
lease_id: lease_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
issued_at_ms,
|
||||
snapshot_generation: Some(snapshot_generation),
|
||||
};
|
||||
let runtime = self.clone();
|
||||
let lease_id = lease_id.clone();
|
||||
let session_id = session_id.clone();
|
||||
let guard = guard.clone();
|
||||
self.lease_completion
|
||||
.run(key, async move { runtime.end_failed_revision_owned(&lease_id, &session_id, &guard, now_ms).await })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn end_failed_revision_owned(
|
||||
&self,
|
||||
lease_id: &HlsAccessLeaseId,
|
||||
session_id: &ProxySessionId,
|
||||
guard: &HlsRevisionRecoveryGuard,
|
||||
now_ms: u64,
|
||||
) -> bool {
|
||||
let _users = self.lease_completion.users().await;
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return false };
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
let ended_at_ms = tuliprox_core::utils::current_time_millis().max(now_ms);
|
||||
let preparation = leases.end_revision_playback_if_current(lease_id, guard, &session, ended_at_ms);
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
let Some(preparation) = preparation else { return false };
|
||||
self.complete_prepared_lease(lease_id, &preparation).await
|
||||
}
|
||||
|
||||
pub(crate) async fn recover_segment_leases_after_fetch(
|
||||
&self,
|
||||
session_id: &ProxySessionId,
|
||||
seq: u64,
|
||||
origin_generation: u64,
|
||||
now_ms: u64,
|
||||
) {
|
||||
let Some(current) = self.sessions.hold_current_proxy_session(session_id).await else { return };
|
||||
let mut leases = self.access_leases.write().await;
|
||||
let session = current.session().read().await;
|
||||
if session.activity.origin_work_generation != origin_generation || session.is_gc_marked_for_removal() {
|
||||
return;
|
||||
}
|
||||
let ids = leases.revision_bound_lease_ids(session_id, seq);
|
||||
let mut endings = Vec::new();
|
||||
for id in ids {
|
||||
if let Some(guard) = leases.revision_recovery_guard(&id, seq) {
|
||||
if leases.recover_revision_binding(&id, &guard, &session, now_ms)
|
||||
== HlsRevisionRecoveryOutcome::ReentryRequired
|
||||
{
|
||||
endings.push((id, guard));
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(session);
|
||||
drop(leases);
|
||||
drop(current);
|
||||
for (id, guard) in endings {
|
||||
self.end_access_lease_for_failed_revision(&id, session_id, &guard, now_ms).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -194,6 +194,7 @@ async fn finalized_signed_url_generations_follow_published_lease_lifetime() {
|
||||
.await
|
||||
.expect("expired lease remains available for lifecycle cleanup");
|
||||
assert_eq!(expired_a.state, HlsAccessLeaseState::Expired);
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_a_id).await;
|
||||
let mut session = session.write().await;
|
||||
assert_eq!(session.transient.finalized_manifest_generation_count(), 1);
|
||||
@@ -309,6 +310,7 @@ async fn same_lease_retains_every_published_finalized_generation_until_removal()
|
||||
.is_some());
|
||||
}
|
||||
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
let session = session.read().await;
|
||||
assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 0);
|
||||
@@ -434,6 +436,7 @@ async fn lease_reconciliation_cannot_resurrect_a_concurrently_removed_binding()
|
||||
)
|
||||
};
|
||||
wait_until_access_lease_store_is_write_locked(&manager).await;
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
let removal_task = {
|
||||
let manager = Arc::clone(&manager);
|
||||
let lease_id = lease_id.clone();
|
||||
|
||||
@@ -1109,6 +1109,7 @@ async fn resource_denial_does_not_destroy_existing_terminal_tail() {
|
||||
);
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
assert_eq!(
|
||||
@@ -1148,6 +1149,7 @@ async fn terminal_tail_protection_is_retained_until_bounded_lease_removal() {
|
||||
);
|
||||
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
}
|
||||
@@ -1188,6 +1190,7 @@ async fn terminal_unavailable_protection_is_retained_until_bounded_lease_removal
|
||||
assert_eq!(denied.state, HlsAccessLeaseState::Denied);
|
||||
assert!(matches!(denied.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_some());
|
||||
manager.bind_lease_user_accounting(&crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users);
|
||||
manager.remove_access_lease(&lease_id).await;
|
||||
assert!(session.read().await.terminal_tail_protection(&lease_id).is_none());
|
||||
}
|
||||
|
||||
@@ -92,3 +92,116 @@ use self::support::{
|
||||
StaleSwitchGeneration, TestOriginServer, CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, SWITCH_MAP_BODY,
|
||||
SWITCH_SEGMENT_BODY,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(
|
||||
clippy::too_many_lines,
|
||||
reason = "Keeps the degraded admission and unchanged-success counterprobe in one scenario."
|
||||
)]
|
||||
async fn retryable_manifest_failure_keeps_optimistic_admission_degraded_until_media_progress() -> std::io::Result<()> {
|
||||
use crate::{
|
||||
media_reserve::{
|
||||
evaluate_startup_admission, HlsReadyTimelineSnapshot, HlsStartupAdmissionDecision,
|
||||
HlsStartupAdmissionInput, HlsStartupAdmissionOriginState,
|
||||
},
|
||||
origin_progress::HlsOriginPathCondition,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
let session = test_session();
|
||||
let context = HlsOriginManifestFetchContext {
|
||||
app_config: test_app_config(),
|
||||
session: Arc::clone(&session),
|
||||
availability_runtime: None,
|
||||
origin_entry: LiveHlsOriginEntry::parse("http://127.0.0.1:9/index.m3u8").unwrap(),
|
||||
headers: axum::http::HeaderMap::new(),
|
||||
client: reqwest::Client::new(),
|
||||
no_redirect_client: reqwest::Client::new(),
|
||||
use_manual_redirects: false,
|
||||
origin_manifest_timeout_ms: 1000,
|
||||
manifest_recovery_burst: tuliprox_core::model::HlsManifestRecoveryBurstConfig::default(),
|
||||
retry_policy: RetryPolicy::default(),
|
||||
recovery_timing_policy: test_recovery_timing_policy(100),
|
||||
acceptance_timing_seed: None,
|
||||
};
|
||||
let manifest = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4,\n0.ts\n#EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n");
|
||||
let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(origin) =
|
||||
tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(
|
||||
&manifest.body,
|
||||
&manifest.final_manifest_url,
|
||||
)
|
||||
else {
|
||||
panic!("normal");
|
||||
};
|
||||
let snapshot = {
|
||||
let mut session = session.write().await;
|
||||
session.apply_origin_manifest(&origin).unwrap();
|
||||
session.origin_control.record_media_progress(10, 4000);
|
||||
let config = tuliprox_core::model::HlsStartupConfig {
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
..Default::default()
|
||||
};
|
||||
let store = Arc::new(crate::SegmentRevisionStore::default());
|
||||
let mut revisions = std::collections::BTreeMap::new();
|
||||
for seq in 0..3 {
|
||||
revisions
|
||||
.insert(seq, store.create(session.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?);
|
||||
}
|
||||
revisions[&0].revision().prefix_available.store(1, std::sync::atomic::Ordering::Release);
|
||||
session.startup = Some(crate::HlsSessionStartup {
|
||||
config: config.clone(),
|
||||
first_data_timeout_ms: 10000,
|
||||
worker: std::sync::Weak::new(),
|
||||
access_leases: Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default())),
|
||||
revisions,
|
||||
policy_fixed: true,
|
||||
store,
|
||||
budget: crate::ProgressiveBudgetManager::new(config),
|
||||
});
|
||||
let rendered = session.render_and_store_manifest(10).unwrap();
|
||||
crate::api::derive_hls_lease_manifest_snapshot(
|
||||
&crate::api::HlsLeaseManifestSnapshotInput::NormalCacheTimeline {
|
||||
session: &session,
|
||||
committed_body: &rendered.body,
|
||||
materialized_body: &rendered.body,
|
||||
stripped_tail_segments: 0,
|
||||
},
|
||||
10,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
};
|
||||
let evaluate = |origin_state| {
|
||||
evaluate_startup_admission(HlsStartupAdmissionInput {
|
||||
manifest: &snapshot,
|
||||
ready_timeline: &HlsReadyTimelineSnapshot { units: Arc::from([]) },
|
||||
origin_state,
|
||||
recovery_trigger_budget: crate::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0),
|
||||
})
|
||||
.decision
|
||||
};
|
||||
assert_eq!(evaluate(HlsStartupAdmissionOriginState::Healthy), HlsStartupAdmissionDecision::Admit);
|
||||
let generation = session.read().await.origin_control.progress_generation;
|
||||
crate::availability::record_retryable_manifest_attempt_failure(
|
||||
&context,
|
||||
generation,
|
||||
&OriginManifestFetchError::RetryableStatus(axum::http::StatusCode::BAD_GATEWAY, None),
|
||||
11,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(session.read().await.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure);
|
||||
assert!(matches!(evaluate(HlsStartupAdmissionOriginState::Degraded), HlsStartupAdmissionDecision::Reject(_)));
|
||||
{
|
||||
let mut session = session.write().await;
|
||||
let timing = build_manifest_refresh_timing(Some(4000), Some(4000), HlsManifestProgress::Unchanged);
|
||||
record_committed_manifest_media_progress(
|
||||
&mut session.origin_control,
|
||||
HlsManifestCommitProgressEvidence::CacheTimeline(timing),
|
||||
Some(4),
|
||||
12,
|
||||
);
|
||||
assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::RetryableFetchFailure);
|
||||
session.origin_control.record_media_progress(13, 4000);
|
||||
assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::ProgressExpected);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -84,6 +84,8 @@ pub struct HlsCacheResponseContext {
|
||||
pub segment_repair: Arc<HlsSegmentRepairManager>,
|
||||
pub qos_meter: Arc<ArcSwapOption<StreamMeterHandle>>,
|
||||
pub media_activity_marker: Option<HlsMediaActivityMarker>,
|
||||
/// Only GET may reserve a lease's progressive opportunity; HEAD waits for metadata.
|
||||
pub request_method: axum::http::Method,
|
||||
pub now_ms: u64,
|
||||
}
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ impl HlsCacheResponseContext {
|
||||
segment_repair,
|
||||
qos_meter: Arc::new(ArcSwapOption::from(qos_meter)),
|
||||
media_activity_marker,
|
||||
request_method: axum::http::Method::GET,
|
||||
now_ms,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,24 +71,16 @@ async fn prepare_hls_revision_response(
|
||||
(Duration::from_secs(startup.config.max_progressive_reader_lifetime_secs.get()), startup.first_data_timeout_ms)
|
||||
};
|
||||
let deadline = deadline.min(remaining_wait_deadline(context.now_ms, wait_ms));
|
||||
let (revision, progressive) = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
let Some(revision) = leases.published_segment_revision(&context.hls_access_lease_id, segment_file.proxy_seq)
|
||||
else {
|
||||
return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing);
|
||||
};
|
||||
let full_request = range.is_none_or(|range| range.as_bytes() == b"bytes=0-");
|
||||
let progressive = revision.revision().key.kind == super::super::SegmentRevisionKind::Raw
|
||||
&& full_request
|
||||
&& leases.try_claim_progressive_startup(
|
||||
&context.hls_access_lease_id,
|
||||
segment_file.proxy_seq,
|
||||
context.now_ms,
|
||||
);
|
||||
(revision, progressive)
|
||||
};
|
||||
let metadata = match revision_metadata(revision.revision(), progressive, deadline).await {
|
||||
Ok(metadata) => metadata,
|
||||
let (revision, metadata) = match resolve_revision_metadata(
|
||||
marker,
|
||||
segment_file,
|
||||
range,
|
||||
context.request_method == axum::http::Method::GET,
|
||||
deadline,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resolved) => resolved,
|
||||
Err(failure) => return HlsResourceServeOutcome::Failure(failure),
|
||||
};
|
||||
let authorized_at_ms = current_time_millis();
|
||||
@@ -154,6 +146,95 @@ fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::I
|
||||
tokio::time::Instant::now() + Duration::from_millis(wait_ms.saturating_sub(elapsed_ms))
|
||||
}
|
||||
|
||||
async fn resolve_revision_metadata(
|
||||
marker: &super::HlsMediaActivityMarker,
|
||||
segment_file: &HlsSegmentFile,
|
||||
range: Option<&HeaderValue>,
|
||||
is_get: bool,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<(crate::SegmentRevisionGuard, Option<CachedSegmentMetadata>), HlsResourceServeFailure> {
|
||||
use crate::lease::HlsRevisionRecoveryOutcome;
|
||||
// Reserve before waiting for recovery or the first body bytes. A concurrent
|
||||
// request must not steal the opportunity from a headers-only first request.
|
||||
let startup_claimed = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
let now_ms = current_time_millis();
|
||||
if !leases.media_identity_is_current(&marker.lease_id, &marker.proxy_session_id, marker.lease_identity, now_ms)
|
||||
{
|
||||
return Err(HlsResourceServeFailure::Expired);
|
||||
}
|
||||
if leases.published_segment_revision(&marker.lease_id, segment_file.proxy_seq).is_none() {
|
||||
return Err(HlsResourceServeFailure::Missing);
|
||||
}
|
||||
let full_request = is_get && range.is_none_or(|range| range.as_bytes() == b"bytes=0-");
|
||||
full_request && leases.try_claim_progressive_startup(&marker.lease_id, segment_file.proxy_seq, now_ms)
|
||||
};
|
||||
let mut last_outcome = None;
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 });
|
||||
}
|
||||
let now_ms = current_time_millis();
|
||||
let outcome = marker
|
||||
.manager
|
||||
.recover_access_lease_revision(
|
||||
&marker.lease_id,
|
||||
&marker.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
segment_file.proxy_seq,
|
||||
now_ms,
|
||||
)
|
||||
.await;
|
||||
if outcome == HlsRevisionRecoveryOutcome::ReentryRequired {
|
||||
return Err(HlsResourceServeFailure::Missing);
|
||||
}
|
||||
let (revision, progressive) = {
|
||||
let mut leases = marker.manager.access_leases().write().await;
|
||||
if !leases.media_identity_is_current(
|
||||
&marker.lease_id,
|
||||
&marker.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
now_ms,
|
||||
) {
|
||||
return Err(HlsResourceServeFailure::Expired);
|
||||
}
|
||||
let revision = leases
|
||||
.published_segment_revision(&marker.lease_id, segment_file.proxy_seq)
|
||||
.ok_or(HlsResourceServeFailure::Missing)?;
|
||||
if outcome == HlsRevisionRecoveryOutcome::RetryPending {
|
||||
if last_outcome != Some(outcome) {
|
||||
let session = marker.session.read().await;
|
||||
log::debug!(
|
||||
"HLS revision recovery: lease={} seq={} generation={} revision={:?} prefix={} exposed={} outcome=retry_pending",
|
||||
crate::safe_hls_access_lease_id(&marker.lease_id), segment_file.proxy_seq,
|
||||
session.activity.origin_work_generation, revision.revision().id(),
|
||||
revision.revision().prefix_available.load(std::sync::atomic::Ordering::Acquire),
|
||||
revision.revision().was_exposed(),
|
||||
);
|
||||
}
|
||||
last_outcome = Some(outcome);
|
||||
drop(leases);
|
||||
tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await;
|
||||
continue;
|
||||
}
|
||||
let progressive = startup_claimed && revision.revision().key.kind == crate::SegmentRevisionKind::Raw;
|
||||
(revision, progressive)
|
||||
};
|
||||
match revision_metadata(revision.revision(), progressive, deadline).await {
|
||||
Ok(metadata) => return Ok((revision, metadata)),
|
||||
Err(HlsResourceServeFailure::PermanentFailed { .. }) => {
|
||||
// A pending revision may fail while this request waits for EOF.
|
||||
// Re-resolve through the same fenced transaction and deadline.
|
||||
if !matches!(revision.revision().state(), SegmentRevisionState::Failed) {
|
||||
return Err(HlsResourceServeFailure::PermanentFailed { status: None });
|
||||
}
|
||||
tokio::time::sleep_until(deadline.min(tokio::time::Instant::now() + Duration::from_millis(25))).await;
|
||||
}
|
||||
Err(failure) => return Err(failure),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A progressive claim streams a publishable pending revision; every other
|
||||
/// request waits for the complete revision file.
|
||||
async fn revision_metadata(
|
||||
@@ -162,11 +243,26 @@ async fn revision_metadata(
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<CachedSegmentMetadata>, HlsResourceServeFailure> {
|
||||
if progressive {
|
||||
return match revision.state() {
|
||||
SegmentRevisionState::Complete(metadata) => Ok(Some(metadata)),
|
||||
SegmentRevisionState::Pending if revision.is_publishable() => Ok(None),
|
||||
_ => Err(HlsResourceServeFailure::PermanentFailed { status: None }),
|
||||
};
|
||||
let mut changes = revision.subscribe();
|
||||
loop {
|
||||
let state = changes.borrow_and_update().clone();
|
||||
match state {
|
||||
SegmentRevisionState::Complete(metadata) => return Ok(Some(metadata)),
|
||||
SegmentRevisionState::Failed => {
|
||||
return Err(HlsResourceServeFailure::PermanentFailed { status: None });
|
||||
}
|
||||
SegmentRevisionState::Pending if revision.can_expose() => return Ok(None),
|
||||
SegmentRevisionState::Pending => {}
|
||||
}
|
||||
// Optimistic successors can be listed before any body byte arrives.
|
||||
// Without replay (or after replay exhaustion) wait for the complete
|
||||
// immutable file instead; never expose a missing prefix as a 404.
|
||||
match tokio::time::timeout_at(deadline, changes.changed()).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(_)) => return Err(HlsResourceServeFailure::PermanentFailed { status: None }),
|
||||
Err(_) => return Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }),
|
||||
}
|
||||
}
|
||||
}
|
||||
match revision.wait_complete(deadline).await {
|
||||
Ok(metadata) => Ok(Some(metadata)),
|
||||
|
||||
@@ -67,6 +67,7 @@ async fn live_media_marker(
|
||||
struct CachedSegmentFixture {
|
||||
_temp_dir: tempfile::TempDir,
|
||||
manager: Arc<HlsProxyManager>,
|
||||
active_users: Arc<tuliprox_session::ActiveUserManager>,
|
||||
segment_cache: Arc<HlsSegmentCache>,
|
||||
session: HlsSessionHandle,
|
||||
proxy_session_id: ProxySessionId,
|
||||
@@ -80,8 +81,15 @@ struct CachedSegmentFixture {
|
||||
|
||||
impl CachedSegmentFixture {
|
||||
async fn new(bytes: Vec<u8>) -> Self {
|
||||
let fixture = Self::without_lease_user_accounting(bytes).await;
|
||||
fixture.manager.bind_lease_user_accounting(&fixture.active_users);
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn without_lease_user_accounting(bytes: Vec<u8>) -> Self {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300));
|
||||
let active_users = crate::HlsCtx::for_test(tuliprox_core::model::Config::default()).active_users;
|
||||
let segment_cache = Arc::clone(manager.segment_cache());
|
||||
let session = manager.get_or_create_session(HlsSessionKey::new(1, "range-stream"), b"secret", 1_000).await;
|
||||
let proxy_session_id = session.read().await.proxy_session_id.clone();
|
||||
@@ -137,6 +145,7 @@ impl CachedSegmentFixture {
|
||||
Self {
|
||||
_temp_dir: temp_dir,
|
||||
manager,
|
||||
active_users,
|
||||
segment_cache,
|
||||
session,
|
||||
proxy_session_id,
|
||||
@@ -202,6 +211,7 @@ async fn publish_revision_fixture(
|
||||
fixture.segment_file.proxy_seq,
|
||||
if raw { crate::SegmentRevisionKind::Raw } else { crate::SegmentRevisionKind::Processed },
|
||||
)?;
|
||||
revision.revision().bind_origin(&*fixture.session.read().await);
|
||||
if raw {
|
||||
let mut replay =
|
||||
crate::progressive_startup::ProgressiveReplay::new(&budget).ok_or_else(|| std::io::Error::other("slot"))?;
|
||||
@@ -283,8 +293,10 @@ async fn publish_revision_fixture(
|
||||
}
|
||||
|
||||
mod admission;
|
||||
mod first_requested;
|
||||
mod http;
|
||||
mod lifecycle;
|
||||
mod policy;
|
||||
mod recovery;
|
||||
mod revision_recovery;
|
||||
mod terminal;
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
//! Startup follows the player's first full GET, including a headers-only successor.
|
||||
use super::*;
|
||||
use crate::sync_ext::MutexExt;
|
||||
|
||||
const BODY: &[u8] = b"prefix-tail";
|
||||
|
||||
struct FirstRequestFixture {
|
||||
cached: CachedSegmentFixture,
|
||||
revisions: std::collections::BTreeMap<u64, crate::SegmentRevisionGuard>,
|
||||
}
|
||||
|
||||
impl FirstRequestFixture {
|
||||
async fn new(mode: shared::model::HlsStartupMode, with_prefix: bool, with_replay: bool) -> std::io::Result<Self> {
|
||||
let cached = CachedSegmentFixture::new(BODY.to_vec()).await;
|
||||
let head = publish_revision_fixture(&cached, true).await?;
|
||||
let mut revisions = std::collections::BTreeMap::from([(12, head)]);
|
||||
{
|
||||
let mut session = cached.session.write().await;
|
||||
session.startup.as_mut().expect("startup").config.mode = mode;
|
||||
let store = Arc::clone(&session.startup.as_ref().expect("startup").store);
|
||||
let budget = Arc::clone(&session.startup.as_ref().expect("startup").budget);
|
||||
for seq in [13, 14] {
|
||||
let mut entry = session.segments[&12].clone();
|
||||
entry.proxy_seq = seq;
|
||||
entry.origin_key.host_local_sequence = seq;
|
||||
entry.origin_key.host_local_index = u32::try_from(seq).expect("fixture sequence");
|
||||
entry.cache_key = SegmentCacheKey::new(cached.proxy_session_id.clone(), seq, "ts");
|
||||
entry.media_start_ms = seq * 4000;
|
||||
entry.status = SegmentCacheStatus::Fetching {
|
||||
priority: crate::SegmentFetchPriority::Demand,
|
||||
started_at_ms: current_time_millis(),
|
||||
};
|
||||
session.segments.insert(seq, entry);
|
||||
let revision = store.create(cached.proxy_session_id.clone(), seq, crate::SegmentRevisionKind::Raw)?;
|
||||
revision.revision().bind_origin(&session);
|
||||
if with_replay {
|
||||
let mut replay = crate::progressive_startup::ProgressiveReplay::new(&budget)
|
||||
.ok_or_else(|| std::io::Error::other("replay slot"))?;
|
||||
if with_prefix {
|
||||
replay.push(b"prefix")?;
|
||||
revision.revision().prefix_available.store(6, Ordering::Release);
|
||||
}
|
||||
*revision.revision().replay.lock_unpoisoned() = Some(replay);
|
||||
}
|
||||
session.startup.as_mut().expect("startup").revisions.insert(seq, revision.clone());
|
||||
revisions.insert(seq, revision);
|
||||
}
|
||||
}
|
||||
let mut leases = cached.manager.access_leases().write().await;
|
||||
let mut snapshot = leases
|
||||
.response_snapshot(&cached.lease_id, &cached.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
let mut segments = snapshot.visible_segments.to_vec();
|
||||
for seq in [13, 14] {
|
||||
let mut segment = segments[0].clone();
|
||||
segment.proxy_seq = seq;
|
||||
segment.uri = format!("{seq}.ts").into();
|
||||
segments.push(segment);
|
||||
}
|
||||
snapshot.visible_segments = Arc::from(segments);
|
||||
snapshot.last_proxy_seq = 14;
|
||||
snapshot.playlist_duration_ms = 12_000;
|
||||
snapshot.last_visible_media_end_ms = 12_000;
|
||||
snapshot.startup_revisions =
|
||||
Some(Arc::new(crate::HlsManifestRevisions { mode, retained_start_seq: 12, revisions: revisions.clone() }));
|
||||
let now = current_time_millis();
|
||||
let guard =
|
||||
leases.prepare_manifest_publication(&cached.lease_id, &cached.proxy_session_id, now).expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(&cached.lease_id, &cached.proxy_session_id, guard, snapshot, now)
|
||||
.is_committed());
|
||||
drop(leases);
|
||||
Ok(Self { cached, revisions })
|
||||
}
|
||||
|
||||
async fn serve(&self, seq: u64, range: Option<&str>) -> HlsResourceServeOutcome {
|
||||
self.serve_method(seq, range, axum::http::Method::GET).await
|
||||
}
|
||||
|
||||
async fn serve_method(&self, seq: u64, range: Option<&str>, method: axum::http::Method) -> HlsResourceServeOutcome {
|
||||
let context = HlsCacheResponseContext {
|
||||
hls_access_lease_id: self.cached.lease_id.clone(),
|
||||
log_identity: self.cached.context.log_identity.clone(),
|
||||
cache_duration_seconds: self.cached.context.cache_duration_seconds,
|
||||
metrics: Arc::clone(&self.cached.context.metrics),
|
||||
segment_repair: Arc::clone(&self.cached.context.segment_repair),
|
||||
qos_meter: Arc::clone(&self.cached.context.qos_meter),
|
||||
media_activity_marker: self.cached.context.media_activity_marker.clone(),
|
||||
request_method: method,
|
||||
now_ms: current_time_millis(),
|
||||
};
|
||||
serve_hls_segment_cache_outcome(
|
||||
Arc::clone(&self.cached.segment_cache),
|
||||
Arc::clone(&self.cached.session),
|
||||
HlsSegmentFile { proxy_seq: seq, extension: "ts".into() },
|
||||
range.map(header),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn prefix(&self, seq: u64) -> std::io::Result<()> {
|
||||
let revision = self.revisions[&seq].revision();
|
||||
revision.replay.lock_unpoisoned().as_mut().expect("replay").push(b"prefix")?;
|
||||
revision.prefix_available.store(6, Ordering::Release);
|
||||
revision.prefix_changed();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete(&self, seq: u64) -> std::io::Result<()> {
|
||||
let cache_key = self.cached.session.read().await.segments[&seq].cache_key.clone();
|
||||
let metadata = self.cached.segment_cache.write_bytes_and_commit(&cache_key, BODY).await?;
|
||||
assert!(self.revisions[&seq].revision().complete(metadata));
|
||||
self.revisions[&seq].revision().replay.lock_unpoisoned().take();
|
||||
self.cached.session.write().await.segments.get_mut(&seq).expect("entry").status = SegmentCacheStatus::Ready {
|
||||
content_length: u64::try_from(BODY.len()).expect("body length"),
|
||||
ready_at_ms: current_time_millis(),
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_successor_streams_before_eof_in_both_raw_startup_modes() -> std::io::Result<()> {
|
||||
for mode in [shared::model::HlsStartupMode::Progressive, shared::model::HlsStartupMode::Optimistic] {
|
||||
let fixture = FirstRequestFixture::new(mode, true, true).await?;
|
||||
let response = ready_response(fixture.serve(13, None).await);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(
|
||||
body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(),
|
||||
Some(b"prefix".as_slice())
|
||||
);
|
||||
assert!(fixture.revisions[&13].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&12].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&13].revision().is_complete());
|
||||
let subsequent = fixture.serve(14, None);
|
||||
tokio::pin!(subsequent);
|
||||
assert!(futures::poll!(subsequent.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
let subsequent = ready_response(subsequent.await);
|
||||
assert!(subsequent.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(subsequent.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
fixture.complete(13).await?;
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"-tail".as_slice()));
|
||||
assert!(body.next().await.is_none());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn headers_only_first_get_keeps_the_claim_until_its_own_prefix_arrives() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
let first = fixture.serve(13, Some("bytes=0-"));
|
||||
let later = fixture.serve(14, None);
|
||||
tokio::pin!(first, later);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(later.as_mut()).is_pending());
|
||||
fixture.prefix(14)?;
|
||||
assert!(futures::poll!(later.as_mut()).is_pending(), "a later GET still waits for EOF");
|
||||
fixture.prefix(13)?;
|
||||
let response = ready_response(first.await);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
assert!(futures::poll!(later.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
assert!(ready_response(later.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn headers_only_first_get_times_out_temporarily_and_preserves_the_lease() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
fixture.cached.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25;
|
||||
let failure = match fixture.serve(13, None).await {
|
||||
HlsResourceServeOutcome::Failure(failure) => failure,
|
||||
HlsResourceServeOutcome::Ready(_) => panic!("the origin has not supplied a prefix"),
|
||||
};
|
||||
let response = crate::response::cache::hls_resource_failure_default_response(failure);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response.headers().contains_key(header::RETRY_AFTER));
|
||||
assert!(fixture
|
||||
.cached
|
||||
.manager
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.cached.lease_id,
|
||||
&fixture.cached.proxy_session_id,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_some());
|
||||
fixture.complete(13).await?;
|
||||
let retry = ready_response(fixture.serve(13, None).await);
|
||||
assert!(retry.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(retry.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_metadata_probe_does_not_consume_the_first_get_opportunity() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
let head = fixture.serve_method(12, None, axum::http::Method::HEAD);
|
||||
tokio::pin!(head);
|
||||
assert!(futures::poll!(head.as_mut()).is_pending(), "HEAD waits for complete metadata");
|
||||
let first_get = ready_response(fixture.serve(13, None).await);
|
||||
assert!(!first_get.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert!(fixture.revisions[&13].revision().was_exposed());
|
||||
assert!(!fixture.revisions[&12].revision().was_exposed());
|
||||
fixture.complete(12).await?;
|
||||
assert!(ready_response(head.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_get_without_replay_or_after_unexposed_replay_exhaustion_waits_for_complete_bytes() -> std::io::Result<()>
|
||||
{
|
||||
for replay in [false, true] {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, replay).await?;
|
||||
let first = fixture.serve(13, None);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
if replay {
|
||||
fixture.revisions[&13].revision().replay_exhausted();
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
}
|
||||
fixture.complete(13).await?;
|
||||
let response = ready_response(first.await);
|
||||
assert!(response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
assert_eq!(response.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), BODY);
|
||||
assert!(!fixture.revisions[&13].revision().was_exposed());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_first_get_consumes_startup_but_a_partial_range_does_not() -> std::io::Result<()> {
|
||||
for partial in [false, true] {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
fixture.complete(12).await?;
|
||||
let first = ready_response(fixture.serve(12, if partial { Some("bytes=2-5") } else { None }).await);
|
||||
assert_eq!(first.status(), if partial { StatusCode::PARTIAL_CONTENT } else { StatusCode::OK });
|
||||
first.into_body().collect().await.map_err(std::io::Error::other)?;
|
||||
let next = fixture.serve(13, None);
|
||||
tokio::pin!(next);
|
||||
if partial {
|
||||
let response = ready_response(next.await);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
} else {
|
||||
assert!(futures::poll!(next.as_mut()).is_pending());
|
||||
fixture.complete(13).await?;
|
||||
assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_uri_does_not_consume_startup_and_cancelled_first_get_does() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, false, true).await?;
|
||||
assert!(matches!(
|
||||
fixture.serve(99, None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing)
|
||||
));
|
||||
{
|
||||
let first = fixture.serve(13, None);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
}
|
||||
fixture.prefix(14)?;
|
||||
let next = fixture.serve(14, None);
|
||||
tokio::pin!(next);
|
||||
assert!(futures::poll!(next.as_mut()).is_pending());
|
||||
fixture.complete(14).await?;
|
||||
assert!(ready_response(next.await).headers().contains_key(header::CONTENT_LENGTH));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_successor_body_failure_is_terminal_for_the_exposed_revision() -> std::io::Result<()> {
|
||||
let fixture = FirstRequestFixture::new(shared::model::HlsStartupMode::Optimistic, true, true).await?;
|
||||
let response = ready_response(fixture.serve(13, None).await);
|
||||
let mut body = response.into_body().into_data_stream();
|
||||
assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
fixture.revisions[&13].revision().fail();
|
||||
assert!(body.next().await.is_some_and(|chunk| chunk.is_err()));
|
||||
assert!(body.next().await.is_none());
|
||||
assert!(matches!(
|
||||
fixture.serve(13, None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing)
|
||||
));
|
||||
assert!(fixture
|
||||
.cached
|
||||
.manager
|
||||
.access_lease_response_snapshot(
|
||||
&fixture.cached.lease_id,
|
||||
&fixture.cached.proxy_session_id,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
@@ -131,7 +131,7 @@ async fn dropped_full_range_segment_body_records_request_without_completion() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segment_failure() -> std::io::Result<()> {
|
||||
async fn optimistic_successor_without_a_retry_ends_only_its_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await;
|
||||
let seq = fixture.segment_file.proxy_seq;
|
||||
let head_seq = seq.checked_sub(1).ok_or_else(|| std::io::Error::other("fixture head"))?;
|
||||
@@ -195,12 +195,17 @@ async fn optimistic_successor_that_fails_after_publication_is_a_permanent_segmen
|
||||
.commit_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, guard, snapshot, now)
|
||||
.is_committed());
|
||||
}
|
||||
// Listing does not expose a revision; only streaming bytes does. The origin
|
||||
// failing a listed successor therefore fails just that one segment.
|
||||
// Listing does not expose bytes. With no usable retry and a terminal fetch
|
||||
// state, the controlled failure transition retires this lease.
|
||||
successor.revision().fail();
|
||||
assert!(matches!(
|
||||
fixture.serve(None).await,
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::PermanentFailed { .. })
|
||||
HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing | HlsResourceServeFailure::Expired)
|
||||
));
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,977 @@
|
||||
use super::*;
|
||||
use crate::lease::HlsRevisionRecoveryOutcome;
|
||||
|
||||
async fn permanent_failure_with_counted_user(fixture: &CachedSegmentFixture) -> std::io::Result<()> {
|
||||
failed_optimistic_revision(fixture, 6).await?;
|
||||
assert!(fixture
|
||||
.manager
|
||||
.activate_access_lease(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
current_time_millis(),
|
||||
crate::HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 },
|
||||
)
|
||||
.await
|
||||
.is_activated());
|
||||
fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None };
|
||||
let lease = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease");
|
||||
let mut user = tuliprox_core::model::ProxyUserCredentials::default();
|
||||
user.username = lease.username.clone();
|
||||
user.max_connections = 1;
|
||||
fixture
|
||||
.active_users
|
||||
.create_user_session(tuliprox_session::CreateUserSessionParams {
|
||||
user: &user,
|
||||
session_token: &lease.user_session_token,
|
||||
virtual_id: 1,
|
||||
provider: "synthetic-provider",
|
||||
stream_url: "http://synthetic.invalid/live.m3u8",
|
||||
addr: &"127.0.0.1:12345".parse().expect("address"),
|
||||
connection_permission: shared::model::UserConnectionPermission::Allowed,
|
||||
connection_kind: Some(tuliprox_session::ConnectionKind::Normal),
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
fixture.active_users.mark_grace_active(&lease.username, &lease.user_session_token).await;
|
||||
let version =
|
||||
fixture.active_users.grace_active_version(&lease.username, &lease.user_session_token).await.expect("version");
|
||||
fixture.active_users.activate_grace_active(&lease.username, &lease.user_session_token, version).await;
|
||||
assert_eq!(fixture.active_users.user_connections(&lease.username).await, 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_releases_the_counted_user_reservation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
assert_eq!(
|
||||
fixture.active_users.connection_permission("test-user", 1, 0).await,
|
||||
shared::model::UserConnectionPermission::Exhausted
|
||||
);
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert_eq!(
|
||||
fixture.active_users.connection_permission("test-user", 1, 0).await,
|
||||
shared::model::UserConnectionPermission::Allowed
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_preserves_another_counted_session_of_the_same_user() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let mut user = tuliprox_core::model::ProxyUserCredentials::default();
|
||||
user.username = "test-user".into();
|
||||
user.max_connections = 2;
|
||||
let token = "other-playback";
|
||||
fixture
|
||||
.active_users
|
||||
.create_user_session(tuliprox_session::CreateUserSessionParams {
|
||||
user: &user,
|
||||
session_token: token,
|
||||
virtual_id: 2,
|
||||
provider: "synthetic-provider",
|
||||
stream_url: "http://synthetic.invalid/other.m3u8",
|
||||
addr: &"127.0.0.1:12346".parse().expect("address"),
|
||||
connection_permission: shared::model::UserConnectionPermission::Allowed,
|
||||
connection_kind: Some(tuliprox_session::ConnectionKind::Normal),
|
||||
socket_bound: false,
|
||||
})
|
||||
.await;
|
||||
fixture.active_users.mark_grace_active(&user.username, token).await;
|
||||
let version = fixture.active_users.grace_active_version(&user.username, token).await.expect("version");
|
||||
fixture.active_users.activate_grace_active(&user.username, token, version).await;
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 2);
|
||||
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 1);
|
||||
assert!(fixture.active_users.release_session_streams_and_counted_reservation(&user.username, token).await);
|
||||
assert_eq!(fixture.active_users.user_connections(&user.username).await, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn normal_lifecycle_completion_uses_the_same_counted_user_release() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let expires = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease")
|
||||
.valid_until_ms
|
||||
+ 1;
|
||||
let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default());
|
||||
fixture
|
||||
.manager
|
||||
.handle_lifecycle_event(
|
||||
&fixture.active_users,
|
||||
&ctx.active_provider,
|
||||
crate::HlsLifecycleEvent {
|
||||
key: crate::HlsLifecycleEventKey::AccessLeaseValidity {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
due_at_ms: expires,
|
||||
},
|
||||
expires,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expires,)
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reentry_completion_survives_request_cancellation_during_cleanup() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
let reader = fixture.session.read().await;
|
||||
{
|
||||
let request = recovery(&fixture);
|
||||
tokio::pin!(request);
|
||||
assert!(futures::poll!(request.as_mut()).is_pending());
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.access_leases().try_read().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup holds its lease fence while waiting for the session writer");
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
// The request deadline drops this future while cleanup is blocked.
|
||||
}
|
||||
drop(reader);
|
||||
let settled = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if fixture.manager.qos().is_empty().await
|
||||
&& fixture.manager.segment_repair().stats().await.windows == 0
|
||||
&& fixture.manager.pending_lease_completions_for_test() == 0
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(settled.is_ok(), "cancelled cleanup left QoS or repair state");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_reentry_waiters_share_completion_when_one_is_cancelled() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
let reader = fixture.session.read().await;
|
||||
let second = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(second);
|
||||
{
|
||||
let first = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(first);
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
}
|
||||
drop(reader);
|
||||
assert!(tokio::time::timeout(Duration::from_secs(1), second).await.expect("owned completion"));
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 0);
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_reentry_job_preserves_new_incarnation_and_its_user_reservation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let mut replacement =
|
||||
leases.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis()).expect("lease");
|
||||
replacement.issued_at_ms += 1;
|
||||
let replacement_issued_at = replacement.issued_at_ms;
|
||||
assert!(leases.prepare_access_lease(replacement));
|
||||
drop(leases);
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 1);
|
||||
assert_eq!(fixture.manager.qos().len().await, 1);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(),)
|
||||
.await
|
||||
.expect("new incarnation")
|
||||
.issued_at_ms,
|
||||
replacement_issued_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_reentry_waiting_for_accounting_preserves_replacement_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::without_lease_user_accounting(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
{
|
||||
let request = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(request);
|
||||
assert!(futures::poll!(request.as_mut()).is_pending());
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 1);
|
||||
}
|
||||
// The owned job outlives its caller while waiting for runtime attachment.
|
||||
// It must revalidate the lease after attachment, before releasing anything.
|
||||
let replacement_issued_at = {
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let mut replacement = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease");
|
||||
replacement.issued_at_ms += 1;
|
||||
let issued_at = replacement.issued_at_ms;
|
||||
assert!(leases.prepare_access_lease(replacement));
|
||||
issued_at
|
||||
};
|
||||
fixture.manager.bind_lease_user_accounting(&fixture.active_users);
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.pending_lease_completions_for_test() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("stale completion settles after runtime attachment");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 1);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("replacement lease")
|
||||
.issued_at_ms,
|
||||
replacement_issued_at
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_lifecycle_and_reentry_cleanup_survive_cancellation_of_all_waiters() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
permanent_failure_with_counted_user(&fixture).await?;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
let (expires_at, guard) = {
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let expires_at = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.valid_until_ms
|
||||
+ 1;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
(expires_at, guard)
|
||||
};
|
||||
let ctx = crate::HlsCtx::for_test(tuliprox_core::model::Config::default());
|
||||
let session_reader = fixture.session.read().await;
|
||||
{
|
||||
let lifecycle = fixture.manager.handle_lifecycle_event(
|
||||
&fixture.active_users,
|
||||
&ctx.active_provider,
|
||||
crate::HlsLifecycleEvent {
|
||||
key: crate::HlsLifecycleEventKey::AccessLeaseValidity {
|
||||
lease_id: fixture.lease_id.clone(),
|
||||
proxy_session_id: fixture.proxy_session_id.clone(),
|
||||
},
|
||||
due_at_ms: expires_at,
|
||||
},
|
||||
expires_at,
|
||||
);
|
||||
let reentry = fixture.manager.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis(),
|
||||
);
|
||||
tokio::pin!(lifecycle, reentry);
|
||||
assert!(futures::poll!(lifecycle.as_mut()).is_pending());
|
||||
assert!(futures::poll!(reentry.as_mut()).is_pending());
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.access_leases().try_read().is_ok() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup waits for the session writer while holding its lease fence");
|
||||
assert_eq!(fixture.manager.pending_lease_completions_for_test(), 2);
|
||||
// Both caller futures are cancelled while their owned jobs are blocked.
|
||||
}
|
||||
drop(session_reader);
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while fixture.manager.pending_lease_completions_for_test() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned cleanup settles after all waiters are cancelled");
|
||||
assert_eq!(fixture.active_users.user_connections("test-user").await, 0);
|
||||
assert!(fixture.manager.qos().is_empty().await);
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 0);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn failed_optimistic_revision(
|
||||
fixture: &CachedSegmentFixture,
|
||||
prefix: u64,
|
||||
) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
let revision = publish_revision_fixture(fixture, true).await?;
|
||||
let mut session = fixture.session.write().await;
|
||||
session.startup.as_mut().expect("startup").config.mode = shared::model::HlsStartupMode::Optimistic;
|
||||
drop(session);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let mut snapshot = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
Arc::make_mut(snapshot.startup_revisions.as_mut().expect("revisions")).mode =
|
||||
shared::model::HlsStartupMode::Optimistic;
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.is_committed());
|
||||
revision.revision().prefix_available.store(prefix, Ordering::Release);
|
||||
revision.revision().fail();
|
||||
drop(leases);
|
||||
let mut session = fixture.session.write().await;
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status = SegmentCacheStatus::Fetching {
|
||||
priority: crate::SegmentFetchPriority::Demand,
|
||||
started_at_ms: current_time_millis(),
|
||||
};
|
||||
Ok(revision)
|
||||
}
|
||||
|
||||
async fn install_completed_retry(fixture: &CachedSegmentFixture) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
let mut session = fixture.session.write().await;
|
||||
let startup = session.startup.as_ref().expect("startup");
|
||||
let retry = startup.store.create(
|
||||
fixture.proxy_session_id.clone(),
|
||||
fixture.segment_file.proxy_seq,
|
||||
crate::SegmentRevisionKind::Raw,
|
||||
)?;
|
||||
retry.revision().bind_origin(&session);
|
||||
let entry = session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry");
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() };
|
||||
assert!(retry.revision().complete(crate::CachedSegmentMetadata {
|
||||
path: fixture.segment_cache.object_path(&entry.cache_key),
|
||||
size: fixture.full_size,
|
||||
}));
|
||||
session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone());
|
||||
Ok(retry)
|
||||
}
|
||||
|
||||
async fn recovery(fixture: &CachedSegmentFixture) -> HlsRevisionRecoveryOutcome {
|
||||
let marker = fixture.context.media_activity_marker.as_ref().expect("marker");
|
||||
fixture
|
||||
.manager
|
||||
.recover_access_lease_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn install_spooled_retry(fixture: &CachedSegmentFixture) -> std::io::Result<crate::SegmentRevisionGuard> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (store, cache_key) = {
|
||||
let session = fixture.session.read().await;
|
||||
(
|
||||
session.startup.as_ref().expect("startup").store.clone(),
|
||||
session.segments[&fixture.segment_file.proxy_seq].cache_key.clone(),
|
||||
)
|
||||
};
|
||||
let retry = store.create(
|
||||
fixture.proxy_session_id.clone(),
|
||||
fixture.segment_file.proxy_seq,
|
||||
crate::SegmentRevisionKind::Raw,
|
||||
)?;
|
||||
let reservation = fixture.segment_cache.reserve_revision_peak(&cache_key, fixture.full_size * 5).await?;
|
||||
let (mut file, pin) =
|
||||
fixture.segment_cache.create_revision_spool_with_budget(&retry.revision().key, Some(&reservation)).await?;
|
||||
let bytes = tokio::fs::read(fixture.segment_cache.object_path(&cache_key)).await?;
|
||||
file.write_all(&bytes).await?;
|
||||
file.flush().await?;
|
||||
drop(file);
|
||||
let path = fixture.segment_cache.object_path(&retry.revision().key);
|
||||
fixture.segment_cache.account_reserved_spool(&path, fixture.full_size, &reservation)?;
|
||||
*retry.revision().file_pin.lock().expect("pin") = Some(pin);
|
||||
assert!(retry.revision().complete(crate::CachedSegmentMetadata { path, size: fixture.full_size }));
|
||||
let mut session = fixture.session.write().await;
|
||||
retry.revision().bind_origin(&session);
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::Ready { content_length: fixture.full_size, ready_at_ms: current_time_millis() };
|
||||
session.startup.as_mut().expect("startup").revisions.insert(fixture.segment_file.proxy_seq, retry.clone());
|
||||
Ok(retry)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_revision_retry_recovers_same_get_and_snapshot_without_playlist_refresh() -> std::io::Result<()> {
|
||||
for prefix in [0, 6, 3_230_293] {
|
||||
let fixture = CachedSegmentFixture::new(b"complete-retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, prefix).await?;
|
||||
let before = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease before");
|
||||
let availability =
|
||||
fixture.manager.access_leases().read().await.availability_evidence_generation(&fixture.proxy_session_id);
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
let response = ready_response(fixture.serve(None).await);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"complete-retry-body".as_slice());
|
||||
let after = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("same lease");
|
||||
let snapshot = after.last_manifest_snapshot.expect("snapshot");
|
||||
let previous_snapshot = before.last_manifest_snapshot.expect("previous snapshot");
|
||||
assert!(snapshot.snapshot_generation > previous_snapshot.snapshot_generation);
|
||||
assert_eq!(snapshot.visible_segments, previous_snapshot.visible_segments);
|
||||
assert_eq!(snapshot.first_proxy_seq, previous_snapshot.first_proxy_seq);
|
||||
assert_eq!(
|
||||
snapshot.startup_revisions.as_ref().expect("revisions").revisions[&fixture.segment_file.proxy_seq],
|
||||
retry
|
||||
);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert_ne!(leases.availability_evidence_generation(&fixture.proxy_session_id), availability);
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(retry));
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("guard");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.is_committed());
|
||||
assert!(!failed.revision().was_exposed());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_pending_is_503_without_ending_the_lease() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
fixture.session.write().await.startup.as_mut().expect("startup").first_data_timeout_ms = 25;
|
||||
let failure = match fixture.serve(None).await {
|
||||
HlsResourceServeOutcome::Failure(failure) => failure,
|
||||
HlsResourceServeOutcome::Ready(_) => panic!("retry must wait"),
|
||||
};
|
||||
let response = crate::response::cache::hls_resource_failure_default_response(failure);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(response.headers().contains_key(header::RETRY_AFTER));
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_some());
|
||||
install_completed_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn waiting_get_resolves_a_retry_that_finishes_before_its_deadline() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
let request = fixture.serve(None);
|
||||
let complete = async {
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
install_completed_retry(&fixture).await
|
||||
};
|
||||
let (response, retry) = tokio::join!(request, complete);
|
||||
retry?;
|
||||
let response = ready_response(response);
|
||||
assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), b"retry-body".as_slice());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exposed_failure_ends_only_the_old_lease_and_cannot_append_retry_bytes() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let old = publish_revision_fixture(&fixture, true).await?;
|
||||
let base_snapshot = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("old lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("old snapshot");
|
||||
let response = ready_response(fixture.serve(None).await);
|
||||
let mut stream = response.into_body().into_data_stream();
|
||||
assert_eq!(stream.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice()));
|
||||
old.revision().fail();
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
assert!(stream.next().await.is_some_and(|chunk| chunk.is_err()));
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReentryRequired);
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::Unchanged);
|
||||
assert!(fixture.manager.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.is_some());
|
||||
assert_eq!(
|
||||
fixture.session.read().await.startup.as_ref().expect("shared startup").revisions
|
||||
[&fixture.segment_file.proxy_seq],
|
||||
retry
|
||||
);
|
||||
let new_id = HlsAccessLeaseId("replacement-viewer".into());
|
||||
let marker = live_media_marker(&fixture.manager, &fixture.session, new_id.clone()).await;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert!(leases.media_identity_is_current(
|
||||
&new_id,
|
||||
&fixture.proxy_session_id,
|
||||
marker.lease_identity,
|
||||
current_time_millis()
|
||||
));
|
||||
let mut snapshot = base_snapshot;
|
||||
snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions {
|
||||
mode: shared::model::HlsStartupMode::Optimistic,
|
||||
retained_start_seq: fixture.segment_file.proxy_seq,
|
||||
revisions: [(fixture.segment_file.proxy_seq, retry)].into_iter().collect(),
|
||||
}));
|
||||
let guard = leases
|
||||
.prepare_manifest_publication(&new_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("new publication");
|
||||
assert!(leases
|
||||
.commit_manifest_publication(&new_id, &fixture.proxy_session_id, guard, snapshot, current_time_millis())
|
||||
.is_committed());
|
||||
drop(leases);
|
||||
let context = HlsCacheResponseContext::new(
|
||||
new_id,
|
||||
test_log_identity(),
|
||||
300,
|
||||
fixture.manager.metrics().clone(),
|
||||
fixture.manager.segment_repair().clone(),
|
||||
None,
|
||||
Some(marker),
|
||||
current_time_millis(),
|
||||
);
|
||||
let response = ready_response(
|
||||
serve_hls_segment_cache_outcome(
|
||||
fixture.segment_cache.clone(),
|
||||
fixture.session.clone(),
|
||||
fixture.segment_file.clone(),
|
||||
None,
|
||||
&context,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.into_body().collect().await.expect("new lease body").to_bytes(), b"retry-body".as_slice());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_fences_generation_snapshot_and_lease_incarnation() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq)
|
||||
.expect("guard");
|
||||
install_completed_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let session = fixture.session.read().await;
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()),
|
||||
HlsRevisionRecoveryOutcome::Unchanged
|
||||
);
|
||||
assert!(leases
|
||||
.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, current_time_millis())
|
||||
.is_none());
|
||||
drop(session);
|
||||
drop(leases);
|
||||
// Replacing the same ID does not let a delayed failure end the new viewer.
|
||||
live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await;
|
||||
assert!(
|
||||
!fixture
|
||||
.manager
|
||||
.end_access_lease_for_failed_revision(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
&guard,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_recovery_commits_once_and_preserves_progressive_quota() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
failed_optimistic_revision(&fixture, 6).await?;
|
||||
{
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
assert!(leases.try_claim_progressive_startup(
|
||||
&fixture.lease_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis()
|
||||
));
|
||||
}
|
||||
install_completed_retry(&fixture).await?;
|
||||
let (first, second) = tokio::join!(recovery(&fixture), recovery(&fixture));
|
||||
assert!(matches!(
|
||||
(first, second),
|
||||
(HlsRevisionRecoveryOutcome::ReplacementBound, HlsRevisionRecoveryOutcome::Unchanged)
|
||||
| (HlsRevisionRecoveryOutcome::Unchanged, HlsRevisionRecoveryOutcome::ReplacementBound)
|
||||
));
|
||||
assert!(!fixture.manager.access_leases().write().await.try_claim_progressive_startup(
|
||||
&fixture.lease_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
current_time_millis()
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn origin_generation_and_mapping_changes_never_install_retired_retry_bytes() -> std::io::Result<()> {
|
||||
for change_generation in [false, true] {
|
||||
let fixture = CachedSegmentFixture::new(b"retired-retry".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
install_completed_retry(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let mut session = fixture.session.write().await;
|
||||
if change_generation {
|
||||
session.activity.origin_work_generation += 1;
|
||||
} else {
|
||||
session.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").origin_key.origin_epoch += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, current_time_millis()),
|
||||
HlsRevisionRecoveryOutcome::ReentryRequired
|
||||
);
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_lease_cannot_be_rebound_or_ended_by_a_stale_recovery_guard() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
install_completed_retry(&fixture).await?;
|
||||
let mut leases = fixture.manager.access_leases().write().await;
|
||||
let guard = leases.revision_recovery_guard(&fixture.lease_id, fixture.segment_file.proxy_seq).expect("guard");
|
||||
let expires = leases
|
||||
.response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.expect("lease")
|
||||
.valid_until_ms;
|
||||
let session = fixture.session.read().await;
|
||||
assert_eq!(
|
||||
leases.recover_revision_binding(&fixture.lease_id, &guard, &session, expires + 1),
|
||||
HlsRevisionRecoveryOutcome::Unchanged
|
||||
);
|
||||
assert!(leases.end_revision_playback_if_current(&fixture.lease_id, &guard, &session, expires + 1).is_none());
|
||||
assert_eq!(leases.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq), Some(failed));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_completion_recovers_before_get_and_final_failure_reuses_manager_cleanup() -> std::io::Result<()> {
|
||||
for succeeds in [false, true] {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let generation = fixture.session.read().await.activity.origin_work_generation;
|
||||
fixture.manager.segment_repair().update_config(tuliprox_core::model::HlsSegmentRepairConfig {
|
||||
max_level: shared::model::HlsSegmentRepairMode::Low,
|
||||
apply_to_first_segments: 1,
|
||||
..Default::default()
|
||||
});
|
||||
fixture.manager.segment_repair().start_access_lease_window(fixture.lease_id.clone()).await;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
let other_id = HlsAccessLeaseId("unaffected-viewer".into());
|
||||
let other = live_media_marker(&fixture.manager, &fixture.session, other_id.clone()).await;
|
||||
fixture.manager.segment_repair().start_access_lease_window(other_id.clone()).await;
|
||||
fixture
|
||||
.manager
|
||||
.qos()
|
||||
.ensure_access_lease(&other_id, &fixture.proxy_session_id, current_time_millis(), None)
|
||||
.await;
|
||||
let retry = if succeeds {
|
||||
Some(install_completed_retry(&fixture).await?)
|
||||
} else {
|
||||
fixture.session.write().await.segments.get_mut(&fixture.segment_file.proxy_seq).expect("entry").status =
|
||||
SegmentCacheStatus::FailedPermanent { failed_at_ms: current_time_millis(), status: None };
|
||||
None
|
||||
};
|
||||
let runtime = fixture.manager.segment_worker_pool().revision_recovery.get().expect("manager recovery runtime");
|
||||
runtime
|
||||
.recover_segment_leases_after_fetch(
|
||||
&fixture.proxy_session_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
generation.wrapping_sub(1),
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
Some(failed)
|
||||
);
|
||||
runtime
|
||||
.recover_segment_leases_after_fetch(
|
||||
&fixture.proxy_session_id,
|
||||
fixture.segment_file.proxy_seq,
|
||||
generation,
|
||||
current_time_millis(),
|
||||
)
|
||||
.await;
|
||||
if succeeds {
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
retry
|
||||
);
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 2);
|
||||
assert_eq!(fixture.manager.qos().len().await, 2);
|
||||
} else {
|
||||
assert!(fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.is_none());
|
||||
assert_eq!(fixture.manager.segment_repair().stats().await.windows, 1);
|
||||
assert_eq!(fixture.manager.qos().len().await, 1);
|
||||
}
|
||||
assert!(fixture.manager.access_leases().write().await.media_identity_is_current(
|
||||
&other_id,
|
||||
&fixture.proxy_session_id,
|
||||
other.lease_identity,
|
||||
current_time_millis(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn playlist_publication_uses_the_same_recovery_and_never_reinstalls_an_older_attempt() -> std::io::Result<()> {
|
||||
let fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let old_snapshot = fixture
|
||||
.manager
|
||||
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("lease")
|
||||
.last_manifest_snapshot
|
||||
.expect("snapshot");
|
||||
let retry = install_completed_retry(&fixture).await?;
|
||||
let guard = fixture
|
||||
.manager
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("publication");
|
||||
assert!(fixture
|
||||
.manager
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
old_snapshot.clone(),
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
assert_eq!(
|
||||
fixture
|
||||
.manager
|
||||
.access_leases()
|
||||
.read()
|
||||
.await
|
||||
.published_segment_revision(&fixture.lease_id, fixture.segment_file.proxy_seq),
|
||||
Some(retry)
|
||||
);
|
||||
// An old request can still contain the failed guard, but it cannot rewind
|
||||
// the successful transaction to that guard or make it complete again.
|
||||
assert!(!failed.revision().complete(crate::CachedSegmentMetadata { path: "retired.ts".into(), size: 1 }));
|
||||
let guard = fixture
|
||||
.manager
|
||||
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
|
||||
.await
|
||||
.expect("publication");
|
||||
assert!(fixture
|
||||
.manager
|
||||
.commit_access_lease_manifest_publication(
|
||||
&fixture.lease_id,
|
||||
&fixture.proxy_session_id,
|
||||
guard,
|
||||
old_snapshot,
|
||||
current_time_millis()
|
||||
)
|
||||
.await
|
||||
.is_committed());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_recovery_cycles_release_old_publication_pins_and_replay_budget() -> std::io::Result<()> {
|
||||
let mut fixture = CachedSegmentFixture::new(b"retry-body".to_vec()).await;
|
||||
for _ in 0..128 {
|
||||
let marker = live_media_marker(&fixture.manager, &fixture.session, fixture.lease_id.clone()).await;
|
||||
fixture.context.media_activity_marker = Some(marker);
|
||||
let failed = failed_optimistic_revision(&fixture, 6).await?;
|
||||
let (store, budget) = {
|
||||
let session = fixture.session.read().await;
|
||||
let startup = session.startup.as_ref().expect("startup");
|
||||
(startup.store.clone(), startup.budget.clone())
|
||||
};
|
||||
let retry = install_spooled_retry(&fixture).await?;
|
||||
assert_eq!(recovery(&fixture).await, HlsRevisionRecoveryOutcome::ReplacementBound);
|
||||
assert_eq!(failed.revision().pin_count(), 1, "no failed guard remains in the lease or snapshot");
|
||||
assert_eq!(budget.usage(), (0, 0));
|
||||
fixture.manager.access_leases().write().await.remove_access_lease(&fixture.lease_id);
|
||||
fixture.session.write().await.startup.as_mut().expect("startup").revisions.clear();
|
||||
drop(failed);
|
||||
drop(retry);
|
||||
let retired = store.retire_unpinned()?;
|
||||
assert_eq!(retired.len(), 2);
|
||||
for revision in retired {
|
||||
fixture.segment_cache.delete_if_inactive(&revision.key).await?;
|
||||
assert!(!fixture.segment_cache.object_path(&revision.key).exists());
|
||||
}
|
||||
assert!(store.retire_unpinned()?.is_empty());
|
||||
assert_eq!(fixture.segment_cache.revision_disk_bytes(), 0);
|
||||
assert_eq!(fixture.segment_cache.remaining_revision_bytes(&fixture.proxy_session_id), 0);
|
||||
assert!(!fixture
|
||||
.segment_cache
|
||||
.has_active_temp_files_for_session_except::<crate::SegmentRevisionKey>(&fixture.proxy_session_id, &[]));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -87,6 +87,7 @@ pub enum HlsSwitchResourceKind {
|
||||
/// Bounded scheduler for live HLS segment demand fetches and prefetches.
|
||||
pub struct HlsSegmentWorkerPool {
|
||||
runtime: ArcSwap<SegmentWorkerRuntime>,
|
||||
pub(crate) revision_recovery: std::sync::OnceLock<Arc<crate::manager::HlsLeaseRuntime>>,
|
||||
access_leases: Arc<RwLock<HlsAccessLeaseStore>>,
|
||||
metrics: Arc<HlsCacheMetrics>,
|
||||
availability_reevaluations: Option<Arc<super::availability_reevaluation::HlsAvailabilityReevaluationCoordinator>>,
|
||||
|
||||
@@ -12,6 +12,7 @@ pub(super) struct SegmentFetchCompletion {
|
||||
pub(super) notifier: Option<Arc<Notify>>,
|
||||
pub(super) scheduled_retry: Option<ScheduledSegmentRetry>,
|
||||
pub(super) evidence_changed_for: Option<ProxySessionId>,
|
||||
pub(super) recovery_generation: Option<u64>,
|
||||
}
|
||||
|
||||
pub(super) struct SegmentFetchCommit {
|
||||
@@ -113,6 +114,7 @@ impl HlsSegmentWorkerPool {
|
||||
notifier,
|
||||
scheduled_retry,
|
||||
evidence_changed_for,
|
||||
recovery_generation: generation_valid.then_some(session.activity.origin_work_generation),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -186,6 +186,7 @@ impl HlsSegmentWorkerPool {
|
||||
) -> Self {
|
||||
Self {
|
||||
runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)),
|
||||
revision_recovery: std::sync::OnceLock::new(),
|
||||
access_leases,
|
||||
metrics,
|
||||
availability_reevaluations,
|
||||
@@ -404,6 +405,20 @@ impl HlsSegmentWorkerPool {
|
||||
let mut session = context.session.write().await;
|
||||
self.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, finished_at_ms)
|
||||
};
|
||||
if let Some(recovery_generation) = completion.recovery_generation {
|
||||
if let Some(recovery) = self.revision_recovery.get() {
|
||||
let session_id = context.session.read().await.proxy_session_id.clone();
|
||||
recovery
|
||||
.recover_segment_leases_after_fetch(
|
||||
&session_id,
|
||||
snapshot.proxy_seq,
|
||||
recovery_generation,
|
||||
finished_at_ms,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
let mut notifier = completion.notifier;
|
||||
if completion.stale_commit_cleanup_reserved {
|
||||
if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await {
|
||||
|
||||
@@ -447,12 +447,13 @@ async fn bind_startup_revision(
|
||||
)));
|
||||
}
|
||||
let decides_policy = startup_fill_decides_policy(&session, snapshot.proxy_seq);
|
||||
revision.revision().bind_origin(&session);
|
||||
let Some(startup) = &mut session.startup else { return Ok(()) };
|
||||
startup.policy_fixed |= decides_policy;
|
||||
startup.revisions.insert(snapshot.proxy_seq, revision.clone());
|
||||
// An optimistic window may become complete when its last successor is
|
||||
// accepted, after the head already has bytes.
|
||||
if startup.config.mode.publishes_pending_successors() && session.last_rendered_manifest.is_none() {
|
||||
if startup.config.mode.publishes_pending_successors() {
|
||||
if let Err(error) = session.render_and_store_manifest(current_time_millis()) {
|
||||
debug!(
|
||||
"HLS optimistic startup render after successor binding failed: proxy_seq={} error={error:?}",
|
||||
|
||||
@@ -250,3 +250,114 @@ async fn optimistic_successor_replay_rejection_keeps_the_session_policy() -> std
|
||||
assert!(prepared_head.raw);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn optimistic_binding_renders_cold_and_retained_sessions_in_both_event_orders() -> std::io::Result<()> {
|
||||
for (retained, head_first) in [(false, false), (false, true), (true, false), (true, true)] {
|
||||
// All origin input is supplied in memory; this probe opens no HTTP connection.
|
||||
let server = super::TestSegmentServer {
|
||||
base_url: "http://127.0.0.1:9".into(),
|
||||
requests: Arc::new(tokio::sync::Mutex::new(Vec::new())),
|
||||
task: tokio::spawn(async {}),
|
||||
};
|
||||
let directory = tempfile::tempdir()?;
|
||||
let policy = SegmentFetchPolicy { max_session_segment_fetches: 3, ..Default::default() };
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
{
|
||||
let mut session = context.session.write().await;
|
||||
if retained {
|
||||
for entry in session.segments.values_mut() {
|
||||
entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 };
|
||||
}
|
||||
session.render_and_store_manifest(1).unwrap();
|
||||
assert_eq!(session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq, 1);
|
||||
}
|
||||
session.apply_origin_manifest(&super::normal_manifest(
|
||||
"#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4\n#EXTINF:4,\n4.ts\n#EXTINF:4,\n5.ts\n#EXTINF:4,\n6.ts\n"
|
||||
)).unwrap();
|
||||
while session.segment_prefetch_queue.pop_next().is_some() {}
|
||||
for entry in session.segments.values_mut() {
|
||||
if matches!(entry.status, SegmentCacheStatus::Queued { .. }) {
|
||||
entry.status = SegmentCacheStatus::Discovered;
|
||||
}
|
||||
}
|
||||
for seq in [4, 5, 6] {
|
||||
assert!(session.queue_segment_fetch_candidate(seq, SegmentFetchPriority::Demand, 10));
|
||||
}
|
||||
}
|
||||
install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await;
|
||||
context.session.write().await.startup.as_mut().unwrap().worker = std::sync::Weak::new();
|
||||
let mut snapshots = std::collections::BTreeMap::new();
|
||||
for _ in 0..3 {
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.unwrap();
|
||||
snapshots.insert(snapshot.proxy_seq, snapshot);
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let head = super::super::prepare_startup_fill(&context, &snapshots[&4], Some(2), deadline)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(format!("{e:?}")))?;
|
||||
if head_first {
|
||||
head.observed.lock().await.observe(b"a").await?;
|
||||
}
|
||||
let mut successors = Vec::new();
|
||||
for seq in [5, 6] {
|
||||
let successor = super::super::prepare_startup_fill(&context, &snapshots[&seq], Some(2), deadline)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(format!("{e:?}")))?;
|
||||
successors.push(successor);
|
||||
}
|
||||
if !head_first {
|
||||
let session = context.session.read().await;
|
||||
assert_eq!(
|
||||
session.last_rendered_manifest.as_ref().map(|manifest| manifest.first_proxy_seq),
|
||||
retained.then_some(1)
|
||||
);
|
||||
drop(session);
|
||||
head.observed.lock().await.observe(b"a").await?;
|
||||
}
|
||||
{
|
||||
let session = context.session.read().await;
|
||||
assert_eq!(crate::HlsManifestRenderer::render(&session, 20).unwrap().first_proxy_seq, 4);
|
||||
let stored = session.last_rendered_manifest.as_ref().unwrap().first_proxy_seq;
|
||||
assert_eq!(stored, 4, "retained={retained} head_first={head_first}");
|
||||
assert!(session
|
||||
.startup
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.revisions
|
||||
.values()
|
||||
.all(|owner| matches!(owner.revision().state(), crate::SegmentRevisionState::Pending)));
|
||||
}
|
||||
// A second body chunk is not another render event.
|
||||
let rendered_at = context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms;
|
||||
head.observed.lock().await.observe(b"b").await?;
|
||||
assert_eq!(context.session.read().await.last_rendered_manifest.as_ref().unwrap().rendered_at_ms, rendered_at);
|
||||
assert!(server.requests.lock().await.is_empty());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retired_generation_cannot_bind_or_render_an_optimistic_revision() -> std::io::Result<()> {
|
||||
let server = spawn_sequence_response_server(Vec::new()).await;
|
||||
let directory = tempfile::tempdir()?;
|
||||
let policy = SegmentFetchPolicy::default();
|
||||
let (worker, context, _) = fetch_context(&server, &directory, &policy).await;
|
||||
install_startup(&context, &worker, shared::model::HlsStartupMode::Optimistic).await;
|
||||
let snapshot = worker.next_fetch_snapshot(&context, 10, &policy).await.expect("snapshot");
|
||||
context.session.write().await.activity.origin_work_generation += 1;
|
||||
let result = super::super::prepare_startup_fill(
|
||||
&context,
|
||||
&snapshot,
|
||||
Some(2),
|
||||
tokio::time::Instant::now() + Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
let session = context.session.read().await;
|
||||
assert!(session.startup.as_ref().expect("startup").revisions.is_empty());
|
||||
assert!(session.last_rendered_manifest.is_none());
|
||||
assert_eq!(context.segment_cache.remaining_revision_bytes(&session.proxy_session_id), 0);
|
||||
assert_eq!(super::temp_cache_files(directory.path()).len(), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -58,10 +58,23 @@ pub struct SegmentRevision {
|
||||
pub prefix_available: AtomicU64,
|
||||
state: watch::Sender<SegmentRevisionState>,
|
||||
exposed: AtomicBool,
|
||||
origin_binding: Mutex<Option<RevisionOriginBinding>>,
|
||||
pub(crate) file_pin: Mutex<Option<super::HlsRevisionFilePin>>,
|
||||
pub(crate) replay: Mutex<Option<super::progressive_startup::ProgressiveReplay>>,
|
||||
}
|
||||
|
||||
/// The origin mapping which produced these bytes, independent of the attempt's
|
||||
/// mutable Fetching/Ready status. A retained startup map alone is not proof that
|
||||
/// a revision still belongs to the session's current origin generation.
|
||||
#[derive(Debug, Clone)]
|
||||
struct RevisionOriginBinding {
|
||||
generation: u64,
|
||||
origin_key: crate::OriginSegmentKey,
|
||||
cache_key: SegmentCacheKey,
|
||||
fetch_ref: Option<crate::OriginSegmentFetchRef>,
|
||||
encryption: Option<crate::HlsSegmentEncryption>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for SegmentRevision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("SegmentRevision")
|
||||
@@ -74,6 +87,32 @@ impl fmt::Debug for SegmentRevision {
|
||||
|
||||
impl SegmentRevision {
|
||||
pub fn id(&self) -> SegmentRevisionId { self.key.id }
|
||||
pub(crate) fn was_exposed(&self) -> bool { self.exposed.load(Ordering::Acquire) }
|
||||
|
||||
pub(crate) fn bind_origin(&self, session: &crate::HlsSession) {
|
||||
if let Some(entry) = session.segments.get(&self.key.proxy_seq) {
|
||||
*self.origin_binding.lock_unpoisoned() = Some(RevisionOriginBinding {
|
||||
generation: session.activity.origin_work_generation,
|
||||
origin_key: entry.origin_key,
|
||||
cache_key: entry.cache_key.clone(),
|
||||
fetch_ref: entry.origin_fetch_ref.clone(),
|
||||
encryption: entry.encryption.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn origin_binding_is_current(&self, session: &crate::HlsSession) -> bool {
|
||||
self.key.proxy_session_id() == &session.proxy_session_id
|
||||
&& self.origin_binding.lock_unpoisoned().as_ref().is_some_and(|binding| {
|
||||
binding.generation == session.activity.origin_work_generation
|
||||
&& session.segments.get(&self.key.proxy_seq).is_some_and(|entry| {
|
||||
entry.origin_key == binding.origin_key
|
||||
&& entry.cache_key == binding.cache_key
|
||||
&& entry.origin_fetch_ref == binding.fetch_ref
|
||||
&& entry.encryption == binding.encryption
|
||||
})
|
||||
})
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pin_count(&self) -> usize { self.pins.load(Ordering::Acquire) }
|
||||
|
||||
@@ -117,13 +156,12 @@ impl SegmentRevision {
|
||||
exposed
|
||||
}
|
||||
|
||||
/// An optimistic successor is listed before its body arrives. If it fails
|
||||
/// before receiving any byte, a new attempt may take over its URI.
|
||||
/// Only client exposure makes failed bytes immutable. Internally buffered
|
||||
/// prefix bytes do not prevent an optimistic retry from taking over the URI.
|
||||
pub(crate) fn is_replaceable_in(&self, mode: HlsStartupMode) -> bool {
|
||||
mode.publishes_pending_successors()
|
||||
&& matches!(*self.state.borrow(), SegmentRevisionState::Failed)
|
||||
&& !self.exposed.load(Ordering::Acquire)
|
||||
&& self.prefix_available.load(Ordering::Acquire) == 0
|
||||
}
|
||||
|
||||
pub(crate) fn replay_exhausted(&self) {
|
||||
@@ -164,8 +202,9 @@ impl SegmentRevision {
|
||||
}
|
||||
|
||||
/// Whether this revision may be listed in a startup window. Optimistic
|
||||
/// startup lists a successor as soon as the origin accepted it; delivery
|
||||
/// of a successor always waits for its complete body.
|
||||
/// startup lists a successor as soon as the origin accepted it. The lease's
|
||||
/// first full request may stream it once replay bytes arrive; subsequent
|
||||
/// requests wait for its complete body.
|
||||
pub fn is_window_publishable(&self, mode: HlsStartupMode) -> bool {
|
||||
if mode.publishes_pending_successors() {
|
||||
!matches!(*self.state.borrow(), SegmentRevisionState::Failed)
|
||||
@@ -261,6 +300,7 @@ impl SegmentRevisionStore {
|
||||
prefix_available: AtomicU64::new(0),
|
||||
state: watch::channel(SegmentRevisionState::Pending).0,
|
||||
exposed: AtomicBool::new(false),
|
||||
origin_binding: Mutex::new(None),
|
||||
file_pin: Mutex::new(None),
|
||||
replay: Mutex::new(None),
|
||||
});
|
||||
|
||||
@@ -280,17 +280,30 @@ reverse_proxy:
|
||||
max_deferred_repairs: 32
|
||||
```
|
||||
|
||||
`progressive` lets the first full segment request consume the existing shared origin fill before origin EOF.
|
||||
`progressive` lets each access lease's first authorized full segment GET consume the existing shared origin
|
||||
fill before origin EOF, regardless of its position in the published playlist. A full request has no Range header
|
||||
or uses `Range: bytes=0-`; HEAD and partial ranges wait for complete metadata/bytes and do not consume this
|
||||
opportunity. The first full request reserves the opportunity before waiting, even if its revision is already
|
||||
complete. Concurrent or subsequent requests, including retries, wait for complete bytes. A cancelled first GET
|
||||
does not re-arm the opportunity. A new access lease has its own opportunity; playlist refreshes and seeks do not
|
||||
reset an existing lease's allowance.
|
||||
The complete session delivers immutable raw TS revisions, including subsequent requests, retries and ranges.
|
||||
Repair and the corruption watchdog continue on the processed cache copy. Raw delivery does not correct source
|
||||
timestamp or transport-stream defects; prefer `first_ready` when a source needs repair.
|
||||
|
||||
`optimistic` behaves like `progressive` and publishes the initial playlist earlier. The head segment needs a
|
||||
body prefix, while listed successors only need an accepted HTTP 200 origin response. Successor fetches start
|
||||
together with the head, within the configured per-session concurrency. A client request for a successor waits
|
||||
for its complete body. If the origin fails a successor after it was listed, only that segment fails for the
|
||||
client; already published media never changes. Replay or disk fallback switches the session to `first_ready`
|
||||
exactly like `progressive`.
|
||||
body prefix, while listed successors only need an accepted HTTP 200 origin response. This head requirement
|
||||
controls playlist publication; it does not determine which segment the player requests first. Successor fetches start
|
||||
together with the head, within the configured per-session concurrency. The lease's first full request may select
|
||||
any published successor. It waits for a replay prefix, then streams before EOF; without replay capacity it waits
|
||||
for the complete revision instead. Other requests wait for complete bodies. If the origin fails after a revision
|
||||
has been exposed, the body errors and recovery requires a new playback entry; bytes already exposed are never
|
||||
silently replaced. A failed unexposed revision can recover through its successful retry. Existing replay and disk
|
||||
fallback rules remain in effect; successors do not change the head's session policy.
|
||||
|
||||
A revision is considered exposed when a progressive response is prepared, before the first body chunk is sent.
|
||||
This conservative boundary applies across leases sharing that revision, even if the first client disconnects
|
||||
before reading any bytes. Listing a revision in a manifest alone does not expose it.
|
||||
|
||||
Fast startup is limited to live, clear MPEG-TS objects without MAP or origin byte ranges. Archive, encrypted,
|
||||
fragmented-MP4 and transient sessions use their existing path. Before publication an incompatible source can fall back to that path.
|
||||
|
||||
@@ -158,12 +158,27 @@ Fix:
|
||||
A `503` with `Retry-After` can be normal during short origin/cache waits, especially near startup or when a requested
|
||||
segment is known but not ready yet.
|
||||
|
||||
An optimistic segment whose origin download failed before any bytes were exposed to a client stays temporarily
|
||||
unavailable while its retry chain is active. Its existing URI returns `503` with `Retry-After` during that wait,
|
||||
instead of a permanent `404`. After a successful retry, the same lease and URI serve the complete replacement;
|
||||
the player does not need to refresh the playlist first. Internally buffered prefix bytes do not count as exposure.
|
||||
If a failed revision was already exposed, Tuliprox ends only the affected lease and its old routes return `404`.
|
||||
A new playback entry can use the shared session. Completed revisions and already exposed bytes are never replaced.
|
||||
The first authorized full segment GET of each lease can claim progressive delivery, including for a published
|
||||
successor. HEAD and partial ranges do not consume that opportunity; subsequent GETs wait for complete bytes.
|
||||
See [startup modes](./shared-hls-configuration.md#optional-startup-modes) for the shared policy.
|
||||
|
||||
The degraded-origin admission check still applies: a retryable manifest failure disables accelerated admission,
|
||||
and an unchanged successful manifest does not clear that condition. Media progress is required to clear it.
|
||||
|
||||
Investigate when it repeats for too long:
|
||||
|
||||
| Log signal | Interpretation |
|
||||
| :--- | :--- |
|
||||
| `HLS segment demand fetch skipped by backpressure` | The session or global fetch limit is saturated. |
|
||||
| `HLS segment temporary failure counted` | The origin segment fetch is failing temporarily. |
|
||||
| `HLS revision recovery` | `replacement_bound` repairs an unexposed failed binding; `reentry_required` ends that exact lease. Includes anonymized identity, revision, sequence, generation, prefix size, and exposure. |
|
||||
| `HLS playlist wait` | A changed wait reason: `fresh_render_required`, `missing_revision_or_head_readiness`, or `admission_rejected`. Includes head readiness and degraded-origin state; repeated identical polls are not logged. |
|
||||
| `HLS manifest marked fresh-commit required after hard fetch failure` | Manifest refresh had a hard failure and needs a fresh commit. |
|
||||
| `HLS terminal manifest failed closed` | A typed lease-local terminal decision could not safely serve live or terminal media. The reason identifies state, bundle, deadline, capacity, or retry failure. |
|
||||
| `HLS terminal commit retry owner failed closed` | The bounded autonomous commit worker reached a typed terminal failure; inspect the reason and preceding recovery/bundle events. |
|
||||
|
||||
Reference in New Issue
Block a user