Merge branch 'develop' into feature/failover

This commit is contained in:
euzu
2026-10-19 18:35:11 +02:00
248 changed files with 14760 additions and 1801 deletions
+2
View File
@@ -64,6 +64,8 @@ backend/app/Cargo.lock
# only exist on a developer's own machine), so it is not project source.
/.claude/
/.mcp.json
AGENTS.md
GOAL.md
# Local refactoring measurements (clippy inventories, cargo metadata, timings).
# These are engineering artifacts of the modularization work, not project docs.
Generated
+22 -21
View File
@@ -1292,7 +1292,7 @@ checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb"
[[package]]
name = "frontend"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"base64 0.23.1",
"blake3",
@@ -3795,7 +3795,7 @@ dependencies = [
[[package]]
name = "shared"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"aes",
"base64 0.23.1",
@@ -4399,7 +4399,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tuliprox"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"aes",
"arc-swap",
@@ -4453,6 +4453,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tokio-stream",
"tokio-tungstenite",
"tokio-util",
"tower",
"tower-http",
@@ -4479,7 +4480,7 @@ dependencies = [
[[package]]
name = "tuliprox-auth"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"axum",
@@ -4502,7 +4503,7 @@ dependencies = [
[[package]]
name = "tuliprox-btree"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"blake3",
"crc32fast",
@@ -4522,7 +4523,7 @@ dependencies = [
[[package]]
name = "tuliprox-config-loader"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"blake3",
@@ -4542,7 +4543,7 @@ dependencies = [
[[package]]
name = "tuliprox-core"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"aes",
"arc-swap",
@@ -4593,7 +4594,7 @@ dependencies = [
[[package]]
name = "tuliprox-curation"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"indexmap",
"log",
@@ -4610,7 +4611,7 @@ dependencies = [
[[package]]
name = "tuliprox-dvr"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"blake3",
@@ -4638,7 +4639,7 @@ dependencies = [
[[package]]
name = "tuliprox-hls"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"async-compression",
@@ -4675,7 +4676,7 @@ dependencies = [
[[package]]
name = "tuliprox-iptv"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -4700,7 +4701,7 @@ dependencies = [
[[package]]
name = "tuliprox-library"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"blake3",
"bytes",
@@ -4727,7 +4728,7 @@ dependencies = [
[[package]]
name = "tuliprox-media-server"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"bytes",
"futures",
@@ -4746,7 +4747,7 @@ dependencies = [
[[package]]
name = "tuliprox-messaging"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"chrono",
@@ -4768,7 +4769,7 @@ dependencies = [
[[package]]
name = "tuliprox-metadata"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"chrono",
@@ -4791,7 +4792,7 @@ dependencies = [
[[package]]
name = "tuliprox-mpegts"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"aes",
"bytes",
@@ -4805,7 +4806,7 @@ dependencies = [
[[package]]
name = "tuliprox-parser"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"chrono",
"chrono-tz",
@@ -4825,7 +4826,7 @@ dependencies = [
[[package]]
name = "tuliprox-processing"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"chrono",
@@ -4864,7 +4865,7 @@ dependencies = [
[[package]]
name = "tuliprox-repository"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"base64 0.23.1",
@@ -4902,7 +4903,7 @@ dependencies = [
[[package]]
name = "tuliprox-session"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"arc-swap",
"axum",
@@ -4932,7 +4933,7 @@ dependencies = [
[[package]]
name = "tuliprox-testkit"
version = "3.3.155"
version = "3.3.156"
dependencies = [
"axum",
"base64 0.23.1",
+1 -9
View File
@@ -3,7 +3,7 @@ members = ["backend/*", "frontend", "shared", "tools/tuliprox-testkit"]
resolver = "2"
[workspace.package]
version = "3.3.155"
version = "3.3.156"
edition = "2021"
rust-version = "1.95.0"
@@ -24,14 +24,6 @@ return_self_not_must_use = "allow"
missing_errors_doc = "allow"
large_futures = "allow"
# Its suggested rewrite of `assert!(x.is_empty())` is
# `assert_eq!(x, [] as [some::crate_root::Path; 0])`, which names a type that
# is usually not in scope at the assertion — which is why `clippy --fix`
# declines to apply it. The panic message improves; every one of the 39 call
# sites gets materially harder to read. Reverse this if the suggestion is
# ever made to use an in-scope type.
assert_is_empty = "allow"
[workspace.dependencies]
shared = { version = "3", path = "shared" }
tuliprox-hls = { version = "3", path = "backend/hls" }
+1
View File
@@ -100,6 +100,7 @@ windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Stora
vergen = { version = "10.0.3", features = ["build"] }
[dev-dependencies]
tokio-tungstenite = "0.29.0"
tuliprox-processing = { workspace = true, features = ["test-support"] }
tuliprox-hls = { workspace = true, features = ["test-support"] }
# The migration and playlist-stream tests build legacy v2 databases and
@@ -195,7 +195,7 @@ async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_pi
.await;
assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -326,7 +326,7 @@ async fn resolve_admission_with_strategies_allows_other_channel_after_recent_evi
let admission = result.admission;
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -458,7 +458,7 @@ async fn resolve_admission_with_strategies_does_not_suppress_different_session_o
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
assert_eq!(grace_mode, None);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -641,7 +641,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same
app_state.connection_manager.release_connection(&hls_addr).await;
assert_eq!(app_state.active_users.user_connections(&user.username).await, 0);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
let mut close_rx = app_state.connection_manager.get_close_connection_channel();
let result = resolve_admission_with_strategies(
@@ -664,7 +664,7 @@ async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same
assert_eq!(admission.permission(), UserConnectionPermission::Allowed);
assert_eq!(grace_mode, None);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
assert_eq!(
tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok),
Some(crate::api::model::CloseConnectionSignal::WithReason(
@@ -128,7 +128,7 @@ async fn create_stream_response_details_preserves_stored_headers_when_fallback_o
socket_bound: false,
})
.await;
assert!(!created.is_empty());
assert_ne!(created.len(), 0);
app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await;
let busy = app_state.active_provider.acquire_exact_connection_with_grace(
@@ -233,7 +233,7 @@ async fn force_provider_stream_response_clears_stored_headers_after_fallback_ope
socket_bound: false,
})
.await;
assert!(!created.is_empty());
assert_ne!(created.len(), 0);
app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await;
let busy = app_state.active_provider.acquire_exact_connection_with_grace(
+2 -2
View File
@@ -777,7 +777,7 @@ mod tests {
let refreshed = refresh_forced_targets(Arc::clone(&current), &sources_with(vec![target(1, "my-target")]));
assert!(!refreshed.enabled, "a reload must not enable a filter the operator never asked for");
assert!(refreshed.target_names.is_empty());
assert_eq!(refreshed.target_names.len(), 0);
}
/// Target ids are positional, so a surviving name must come back with
@@ -808,6 +808,6 @@ mod tests {
let refreshed = refresh_forced_targets(forced(&["gone"]), &sources_with(vec![target(1, "other")]));
assert!(!refreshed.enabled, "an empty filter must not be left enabled: it would process nothing");
assert!(refreshed.target_names.is_empty());
assert_eq!(refreshed.target_names.len(), 0);
}
}
@@ -21,7 +21,7 @@ use serde::Deserialize;
use std::{str::FromStr, sync::Arc};
use tuliprox_hls::api::{
finite_hls_immutable_media_response, resolve_hls_standalone_custom_segment, HlsAccessLeaseId,
HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError,
HlsMediaActivityCommitOutcome, HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError,
};
use url::form_urlencoded;
@@ -286,7 +286,9 @@ async fn cvs_standalone_hls_segment_response(
asset_fingerprint,
index,
now_ms,
) {
)
.await
{
Ok(access) => access,
Err(
HlsStandaloneCustomSegmentError::InvalidIndex
@@ -299,13 +301,28 @@ async fn cvs_standalone_hls_segment_response(
{
return *response;
}
finite_hls_immutable_media_response(
access.bytes,
let response = finite_hls_immutable_media_response(
access.bytes.clone(),
headers.get(header::RANGE),
HLS_CVS_CONTENT_TYPE,
HLS_CVS_IMMUTABLE_CACHE_CONTROL,
method == Method::HEAD,
)
);
if response.status().is_success() {
match app_state
.hls
.proxy
.record_standalone_custom_segment_activity(&access_lease, &access, current_time_millis())
.await
{
HlsMediaActivityCommitOutcome::Committed => {}
HlsMediaActivityCommitOutcome::StaleLeaseIdentity => return StatusCode::NOT_FOUND.into_response(),
HlsMediaActivityCommitOutcome::DeferredLockContention => {
return super::hls_terminal_response::hls_temporary_resource_unavailable_response(1_000)
}
}
}
response
}
async fn validate_hls_standalone_custom_access(
@@ -1,4 +1,5 @@
use super::{
super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
clear_hls_provisioning_handoff_consumer, duration_to_millis_saturating,
effective_hls_url_failover_provider_for_fetch_url, hls_access_lease_timing_for_session, hls_access_lease_ttl_ms,
hls_availability_reevaluation_registration_failure_response, hls_cache_configured,
@@ -29,8 +30,7 @@ use tuliprox_hls::api::{
safe_proxy_session_id, safe_session_key, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeasePendingDeadline,
HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccountBindingProtection, HlsManifestAcceptanceDirective,
HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, HlsOriginIoContext, HlsOriginWorkClass,
HlsPostRefreshRuntime, HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry,
OriginRefreshRequest, ProxySessionId, RetryPolicy,
HlsRuntimeCustomTailReason, HlsSessionHandle, HlsSessionStoreOutcome, LiveHlsOriginEntry, ProxySessionId,
};
#[allow(clippy::too_many_arguments)]
@@ -251,13 +251,10 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
prepared_origin.runtime_provider_config.clone(),
)?;
{
let mut session_guard = session.write().await;
let session_guard = session.read().await;
if session_guard.is_gc_marked_for_removal() {
return Some(hls_canonical_retry_after_response());
}
if prepared_origin.origin_account_binding_to_store.is_some() {
session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store);
}
}
mark_hls_authorized_manifest_access(app_state, &session, now_ms).await;
let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else(
@@ -298,7 +295,7 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
(HlsManifestAcceptanceDirective::none(), Some(owner_key))
}
HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => {
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
drop(prepared_origin.preacquired_origin_account_handle);
return Some(hls_canonical_retry_after_response());
}
};
@@ -317,7 +314,6 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
};
let origin_policy = hls_effective_origin_acquire_policy(&session).await;
let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone();
let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle;
let mut origin_io = HlsOriginIoContext {
ctx: hls_ctx.clone(),
@@ -328,49 +324,37 @@ pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest
reservation_ttl_secs,
preacquired_provider_handle: None,
started_generation: None,
prepared_account: None,
playback_kind: session.read().await.origin_playback_kind(),
account_runtime: session.read().await.origin_account_runtime.clone(),
};
if availability_reevaluation_owner_key.is_none() {
if let Some(provider_handle) = preacquired_provider_handle.take() {
origin_io = origin_io.with_preacquired_provider_handle(provider_handle);
origin_io = origin_io.with_managed_provider_handle(provider_handle);
}
}
let refresh_request = OriginRefreshRequest {
app_config: Arc::clone(&app_state.app_config),
session: Arc::clone(&session),
origin_entry,
headers,
origin_provider_session_headers,
client: app_state.http_clients.default.load().as_ref().clone(),
no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(),
use_manual_redirects: app_state.should_use_manual_redirects(),
segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()),
hls_proxy: Arc::clone(&app_state.hls.proxy),
segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()),
segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()),
map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()),
origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(),
manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(),
strip: app_state.hls.proxy.strip().clone(),
retry_policy: RetryPolicy::default(),
reverse_proxy_rewrite_secret: rewrite_secret.to_vec(),
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
manifest_commit_requirement,
fresh_manifest_requirement_generation: None,
acceptance_directive,
access_lease_id: Some(access_lease_id.clone()),
disabled_headers: app_state.get_disabled_headers(),
now_ms,
origin_io: Some(origin_io),
post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }),
};
let refresh_request = hls_origin_refresh_request(
app_state,
&session,
tuliprox_hls::api::HlsPreparedOriginAccount { entry: origin_entry, snapshot: prepared_origin.snapshot },
HlsOriginRefreshOptions {
headers,
origin_io,
manifest_commit_requirement,
acceptance_directive,
access_lease_id: Some(access_lease_id.clone()),
rewrite_secret: rewrite_secret.to_vec(),
now_ms,
},
);
let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused {
refresh_ordering
} else {
HlsManifestRefreshOrdering::Background
};
if let Some(owner_key) = availability_reevaluation_owner_key {
app_state.connection_manager.release_provider_handle(preacquired_provider_handle);
drop(preacquired_provider_handle);
touch_initial_manifest_access_lease_window(
app_state,
access_lease_id,
@@ -217,7 +217,7 @@ pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_han
return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable);
}
match &lease.playback_mode {
HlsLeasePlaybackMode::TerminalTail(_) => {
HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalCustomResponse(_) => {
let Some(response) =
hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id)
else {
@@ -516,6 +516,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_plans(
pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry(
plan: SharedHlsProvisioningSegmentPlan,
content_length: u64,
media_start_ms: u64,
duration_ms: u64,
now_ms: u64,
) -> SegmentEntry {
@@ -531,6 +532,7 @@ pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry(
host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX),
},
proxy_seq: plan.proxy_seq,
media_start_ms,
duration_ms,
proxy_file_ext: "ts".to_string(),
content_type: "video/mp2t".to_string(),
@@ -636,10 +638,13 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_h
session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq);
}
session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq);
let media_start_ms = session_guard.next_media_start_ms;
session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms);
session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1));
session_guard
.segments
.insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms));
session_guard.segments.insert(
plan.proxy_seq,
shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms),
);
inserted = true;
}
if inserted {
@@ -682,10 +687,12 @@ pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline
session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq);
}
session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq);
let media_start_ms = session_guard.next_media_start_ms;
session_guard.next_media_start_ms = media_start_ms.saturating_add(duration_ms);
session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1));
session_guard.segments.insert(
plan.proxy_seq,
shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms),
shared_hls_provisioning_segment_entry(plan, content_length, media_start_ms, duration_ms, now_ms),
);
}
session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS);
+3 -1
View File
@@ -110,10 +110,12 @@ pub(super) struct HlsProxyResourcePathParams {
mod catchup;
mod manifest;
mod origin_account_runtime;
mod origin_refresh_request;
mod owner_token;
mod segment;
mod session;
use self::origin_account_runtime::prepare_hls_origin_runtime;
pub(in crate::api) use catchup::*;
pub(in crate::api) use manifest::*;
pub(in crate::api) use owner_token::*;
@@ -0,0 +1,719 @@
#![allow(clippy::wildcard_imports)]
use super::{
origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
session::*,
*,
};
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
pub(super) async fn prepare_hls_origin_runtime(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
proxy_session_id: &ProxySessionId,
fingerprint: &Fingerprint,
connection_kind: crate::api::model::ConnectionKind,
priority: i8,
work_kind: HlsOriginWorkKind,
work_class: HlsOriginWorkClass,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
{
let current = session.read().await;
if current.origin_source.input_id != input.id
|| (!current.origin_source.input_name.is_empty() && current.origin_source.input_name != input.name)
|| current.origin_account_binding.as_ref().is_some_and(|binding| binding.input_name != input.name)
{
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
reason: hls_no_account_reason_for_binding(current.origin_account_binding.as_ref()),
});
}
}
advance_hls_origin_account_lifecycle(app_state, session, now_ms).await;
// Healthy preparation does not queue behind account replacement. Preserve the manifest
// singleflight boundary when many clients arrive while a qualified window awaits media.
if let Some(current_input) =
app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name)
{
let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await);
let binding = observed.binding.clone();
if let Some(binding) = binding.filter(|binding| {
binding.is_active()
&& hls_origin_account_status(&app_state.hls_ctx(), binding) == HlsOriginAccountStatus::Known
}) {
let prepared = prepared_hls_origin_runtime_for_known_binding(
app_state,
&current_input,
raw_request_url,
session_entry_url,
&binding,
)
.await;
let mut current = session.write().await;
if !current.is_gc_marked_for_removal() && observed.matches_session(&current) {
if current.origin_account_runtime.is_none() {
current.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime {
app_state: Arc::downgrade(app_state),
input_id: input.id,
input_name: Arc::clone(&input.name),
raw_request_url: raw_request_url.to_string(),
entry_url: session_entry_url.to_string(),
fingerprint: fingerprint.clone(),
}));
}
return Ok(PreparedHlsOriginRuntime::published(
prepared,
HlsOriginAccountSnapshot::from_session(&current),
));
}
}
}
let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock);
let _prepare_guard = prepare_lock.lock().await;
let Some(input) = app_state.app_config.get_input_by_id(input.id).filter(|current| current.name == input.name)
else {
let mut session = session.write().await;
if session.origin_account_rebind.is_allowed_now(now_ms) {
if let Some(binding) = session.origin_account_binding.clone() {
session.origin_account_rebind.mark_attempt_started(binding.account_name, now_ms);
session.origin_account_rebind.mark_failed(now_ms);
}
}
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
reason: hls_no_account_reason_for_binding(session.origin_account_binding.as_ref()),
});
};
session.write().await.origin_account_runtime = Some(Arc::new(AppHlsOriginAccountRuntime {
app_state: Arc::downgrade(app_state),
input_id: input.id,
input_name: Arc::clone(&input.name),
raw_request_url: raw_request_url.to_string(),
entry_url: session_entry_url.to_string(),
fingerprint: fingerprint.clone(),
}));
let input = input.as_ref();
advance_hls_origin_account_lifecycle(app_state, session, now_ms).await;
let observed = HlsOriginAccountSnapshot::from_session(&*session.read().await);
let previous = observed.binding.clone();
let mut prepared = prepare_hls_origin_runtime_locked(
app_state,
session,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
work_kind,
work_class,
now_ms,
)
.await?;
let snapshot = publish_hls_origin_account_with_status(app_state, session, |current| {
let snapshot = if let Some(snapshot) = prepared.prepared_snapshot.take() {
if current.is_gc_marked_for_removal() || !snapshot.matches_session(current) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
snapshot
} else {
if current.is_gc_marked_for_removal() || !observed.matches_session(current) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
if let Some(mut binding) = prepared.binding_to_publish.take() {
binding.generation = previous.as_ref().map_or(0, |previous| previous.generation.saturating_add(1));
binding.provider_binding_tag = prepared
.preacquired_origin_account_handle
.as_ref()
.and_then(tuliprox_session::ManagedProviderHandle::handle)
.and_then(|handle| handle.binding_tag);
if previous.as_ref().is_some_and(|previous| previous.account_name != binding.account_name) {
current.handoff_origin_account(binding);
} else {
current.replace_origin_account_binding(Some(binding));
}
}
HlsOriginAccountSnapshot::from_session(current)
};
Ok(snapshot)
})
.await?;
Ok(PreparedHlsOriginRuntime::published(prepared, snapshot))
}
// Lock order: viewer rows, then HLS state. Publication and projection are synchronous once
// both locks are acquired: cancellation cannot publish an account without publishing its UI rows.
pub(super) async fn publish_hls_origin_account_with_status(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
publish: impl FnOnce(
&mut tuliprox_hls::api::HlsSession,
) -> Result<HlsOriginAccountSnapshot, HlsOriginRuntimeAcquireError>,
) -> Result<HlsOriginAccountSnapshot, HlsOriginRuntimeAcquireError> {
let shared_stream_id = hls_cache_shared_stream_id(&session.read().await.proxy_session_id);
let mut projection = app_state.active_users.lock_shared_stream_provider_projection(shared_stream_id).await;
let mut current = session.write().await;
let snapshot = publish(&mut current)?;
if !snapshot.matches_session(&current) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
if let Some(binding) = snapshot.binding.as_ref().filter(|binding| binding.is_active()) {
projection.update_provider(&binding.account_name);
}
Ok(snapshot)
}
async fn advance_hls_origin_account_lifecycle(app_state: &Arc<AppState>, session: &HlsSessionHandle, now_ms: u64) {
promote_elapsed_hls_account_overlaps(app_state, now_ms).await;
detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await;
reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await;
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn prepare_hls_origin_runtime_locked(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
proxy_session_id: &ProxySessionId,
fingerprint: &Fingerprint,
connection_kind: crate::api::model::ConnectionKind,
priority: i8,
work_kind: HlsOriginWorkKind,
work_class: HlsOriginWorkClass,
now_ms: u64,
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
let playback_kind = hls_session_origin_playback_kind(session).await;
let existing_binding = session.read().await.origin_account_binding.clone();
let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached);
if reacquire_detached_binding && !session.read().await.origin_account_rebind.is_allowed_now(now_ms) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref());
if reacquire_detached_binding {
log_hls_origin_binding_reacquire_started(session, work_kind).await;
}
if let Some(binding) = existing_binding {
if binding.is_active()
|| !matches!(hls_origin_account_status(&app_state.hls_ctx(), &binding), HlsOriginAccountStatus::Known)
{
match hls_origin_account_status(&app_state.hls_ctx(), &binding) {
stale_status @ (HlsOriginAccountStatus::Missing
| HlsOriginAccountStatus::Expired
| HlsOriginAccountStatus::Blocked) => {
return rebind_hls_origin_account(
app_state,
session,
input,
raw_request_url,
session_entry_url,
&binding,
stale_status,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await;
}
HlsOriginAccountStatus::Known => {
return Ok(prepared_hls_origin_runtime_for_known_binding(
app_state,
input,
raw_request_url,
session_entry_url,
&binding,
)
.await);
}
}
}
}
match prepare_hls_origin_runtime_with_new_account(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
false,
work_kind,
work_class,
playback_kind,
now_ms,
)
.await
{
Ok(prepared) => {
if reacquire_detached_binding {
if let Some(binding) = prepared.binding_to_publish.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)),
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
}
if work_class.allows_speculative_overlap() {
if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await
{
if reacquire_detached_binding {
if let Some(binding) = prepared.binding_to_publish.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
if let Ok(prepared) = prepare_hls_speculative_origin_runtime(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await
{
if reacquire_detached_binding {
if let Some(binding) = prepared.binding_to_publish.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
} else {
debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value());
}
if work_class.allows_grace() {
match prepare_hls_origin_runtime_with_new_account(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
true,
work_kind,
work_class,
playback_kind,
now_ms,
)
.await
{
Ok(prepared) => {
if reacquire_detached_binding {
if let Some(binding) = prepared.binding_to_publish.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => {
return Err(HlsOriginRuntimeAcquireError::Fatal(status))
}
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
}
} else {
debug!(
"HLS origin account grace skipped: work_class={} reason=background-origin-work",
work_class.as_log_value()
);
}
if reacquire_detached_binding {
log_hls_origin_binding_reacquire_failed(session, "no-account-available").await;
}
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason })
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_lines)]
pub(super) async fn rebind_hls_origin_account(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
stale_binding: &HlsOriginAccountBinding,
stale_status: HlsOriginAccountStatus,
fingerprint: &Fingerprint,
connection_kind: crate::api::model::ConnectionKind,
priority: i8,
playback_kind: PlaybackKind,
now_ms: u64,
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
{
let mut session_guard = session.write().await;
if !session_guard.origin_account_rebind.is_allowed_now(now_ms) {
debug!(
"HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms);
}
let safe_proxy_session = {
let session_guard = session.read().await;
safe_proxy_session_id(&session_guard.proxy_session_id)
};
debug!(
"HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}",
safe_proxy_session,
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
app_state.active_provider.clear_identified_provider_reservation(
&stale_binding.session_owner,
&stale_binding.account_name,
stale_binding.provider_binding_tag,
);
let expected_work_generation = {
let mut session_guard = session.write().await;
if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| {
binding.account_name == stale_binding.account_name
&& binding.session_owner == stale_binding.session_owner
&& binding.provider_binding_tag == stale_binding.provider_binding_tag
}) {
binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms);
debug!(
"HLS origin binding detached: proxy_session={} account={} reason={}",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason()
);
}
session_guard.invalidate_queued_origin_work();
session_guard.activity.origin_work_generation
};
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
&input.name,
&fingerprint.addr,
false,
priority,
connection_kind,
Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)),
) else {
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await;
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
reason: hls_no_account_reason_for_binding(Some(stale_binding)),
});
};
let managed_handle =
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
let Some(provider_config) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_config())
else {
drop(managed_handle);
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(fetch_url) = resolve_hls_origin_fetch_url(
&app_state.app_config,
input,
raw_request_url,
session_entry_url,
Some(&provider_config),
)
.await
else {
drop(managed_handle);
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(new_account_name) = managed_handle.handle().and_then(|handle| handle.allocation.get_provider_name())
else {
drop(managed_handle);
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let mut new_binding = HlsOriginAccountBinding::rebound(
Arc::clone(&input.name),
new_account_name,
stale_binding.session_owner.clone(),
stale_binding.generation.saturating_add(1),
now_ms,
);
new_binding.provider_binding_tag = managed_handle.handle().and_then(|handle| handle.binding_tag);
let prepared_snapshot = publish_hls_origin_account_with_status(app_state, session, |session_guard| {
if !publish_hls_origin_account_rebind(
session_guard,
stale_binding,
expected_work_generation,
new_binding.clone(),
) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
Ok(HlsOriginAccountSnapshot::from_session(session_guard))
})
.await?;
debug!(
"HLS origin account rebound: old_account={} new_account={}",
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
sanitize_sensitive_info(new_binding.account_name.as_ref())
);
Ok(HlsOriginRuntimeAllocation {
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
raw_request_url,
session_entry_url,
&fetch_url,
),
runtime_provider_config: Some(provider_config),
fetch_url,
binding_to_publish: None,
prepared_snapshot: Some(prepared_snapshot),
preacquired_origin_account_handle: Some(managed_handle),
})
}
// Resolution may suspend after detaching. Publish only into the same account and work generation.
fn publish_hls_origin_account_rebind(
session: &mut tuliprox_hls::api::HlsSession,
expected: &HlsOriginAccountBinding,
expected_work_generation: u64,
replacement: HlsOriginAccountBinding,
) -> bool {
if session.is_gc_marked_for_removal()
|| session.activity.origin_work_generation != expected_work_generation
|| !session.origin_account_binding.as_ref().is_some_and(|current| {
current.input_name == expected.input_name
&& current.account_name == expected.account_name
&& current.session_owner == expected.session_owner
&& current.generation == expected.generation
&& current.provider_binding_tag == expected.provider_binding_tag
})
{
return false;
}
session.handoff_origin_account(replacement);
session.origin_account_rebind.mark_success();
true
}
pub(super) async fn mark_hls_origin_rebind_failed(
session: &HlsSessionHandle,
stale_binding: &HlsOriginAccountBinding,
now_ms: u64,
reason: &str,
) {
let mut session_guard = session.write().await;
session_guard.origin_account_rebind.mark_failed(now_ms);
debug!(
"HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
}
use std::{future::Future, pin::Pin, sync::Weak};
use tuliprox_hls::api::{
trigger_origin_refresh_sync, HlsBoundAccountAcquireErrorKind, HlsManifestAcceptanceDirective,
HlsOriginAccountRuntime, HlsOriginAccountSnapshot, HlsOriginAccountWork, HlsPreparedOriginAccount,
LiveHlsOriginEntry,
};
struct AppHlsOriginAccountRuntime {
app_state: Weak<AppState>,
input_id: u16,
input_name: Arc<str>,
raw_request_url: String,
entry_url: String,
fingerprint: Fingerprint,
}
impl std::fmt::Debug for AppHlsOriginAccountRuntime {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AppHlsOriginAccountRuntime").field("input_id", &self.input_id).finish_non_exhaustive()
}
}
impl HlsOriginAccountRuntime for AppHlsOriginAccountRuntime {
#[allow(clippy::too_many_lines)]
fn prepare<'a>(
&'a self,
session: &'a HlsSessionHandle,
work: HlsOriginAccountWork,
class: HlsOriginWorkClass,
) -> Pin<Box<dyn Future<Output = Result<HlsPreparedOriginAccount, HlsBoundAccountAcquireErrorKind>> + Send + 'a>>
{
Box::pin(async move {
let app_state = self.app_state.upgrade().ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?;
let input = app_state
.app_config
.get_input_by_id(self.input_id)
.filter(|input| input.name == self.input_name)
.ok_or(HlsBoundAccountAcquireErrorKind::Missing)?;
let (proxy_id, policy, unavailable) = {
let session = session.read().await;
(
session.proxy_session_id.clone(),
session.effective_origin_acquire_policy_or_default(),
session.is_gc_marked_for_removal(),
)
};
if unavailable
|| !app_state
.hls
.proxy
.segment_worker_pool()
.access_leases()
.write()
.await
.has_usable_access_lease_for_session(&proxy_id, current_time_millis())
{
return Err(HlsBoundAccountAcquireErrorKind::Unavailable);
}
let prepared = prepare_hls_origin_runtime(
&app_state,
session,
&input,
&self.raw_request_url,
&self.entry_url,
&proxy_id,
&self.fingerprint,
policy.connection_kind,
policy.priority,
if work == HlsOriginAccountWork::Manifest {
HlsOriginWorkKind::Manifest
} else {
HlsOriginWorkKind::Resource
},
class,
current_time_millis(),
)
.await
.map_err(|_| HlsBoundAccountAcquireErrorKind::Unavailable)?;
// Drop releases unused allocations synchronously, including cancellation after prepare.
let managed = prepared.preacquired_origin_account_handle;
let entry = LiveHlsOriginEntry::parse_with_provider_configs(
&prepared.fetch_url,
prepared.url_failover_provider,
prepared.runtime_provider_config,
)
.ok_or(HlsBoundAccountAcquireErrorKind::Unavailable)?;
let snapshot = prepared.snapshot;
let (pending, origin_headers) = {
let current = session.read().await;
if current.is_gc_marked_for_removal() || !snapshot.matches_session(&current) {
return Err(HlsBoundAccountAcquireErrorKind::StoreRace);
}
(current.origin_account_manifest_pending.is_some(), current.origin_request_headers.clone())
};
drop(managed);
if work == HlsOriginAccountWork::Media && pending {
let origin_io = HlsOriginIoContext {
ctx: app_state.hls_ctx(),
client_addr: self.fingerprint.addr,
allow_grace: class.allows_grace(),
priority: policy.priority,
connection_kind: policy.connection_kind,
reservation_ttl_secs: session.read().await.account_overlap_timing().reservation_ttl_secs(),
preacquired_provider_handle: None,
started_generation: None,
prepared_account: None,
playback_kind: session.read().await.origin_playback_kind(),
account_runtime: session.read().await.origin_account_runtime.clone(),
};
let manifest_commit_requirement = tuliprox_hls::api::hls_manifest_commit_requirement(
session,
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
None,
false,
current_time_millis(),
)
.await;
let request = hls_origin_refresh_request(
&app_state,
session,
HlsPreparedOriginAccount { entry: entry.clone(), snapshot: snapshot.clone() },
HlsOriginRefreshOptions {
headers: origin_headers,
origin_io,
manifest_commit_requirement,
acceptance_directive: HlsManifestAcceptanceDirective::none(),
access_lease_id: None,
rewrite_secret: app_state.get_encrypt_secret().to_vec(),
now_ms: current_time_millis(),
},
);
trigger_origin_refresh_sync(request).await;
if session.read().await.origin_account_manifest_pending.is_some() {
return Err(HlsBoundAccountAcquireErrorKind::Unavailable);
}
}
Ok(HlsPreparedOriginAccount { entry, snapshot })
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use tuliprox_hls::api::{HlsSession, HlsSessionKey};
#[test]
fn rebind_publication_rejects_superseded_work_binding_and_gc() {
let mut session = HlsSession::new(HlsSessionKey::new(1, "publication"), b"secret", 0);
let expected =
HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("expired"), &session.proxy_session_id, 0);
session.replace_origin_account_binding(Some(expected.clone()));
let replacement = HlsOriginAccountBinding::rebound(
Arc::from("input"),
Arc::from("replacement"),
expected.session_owner.clone(),
1,
1,
);
session.invalidate_queued_origin_work();
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 0, replacement.clone()));
assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name, expected.account_name);
assert!(session.origin_account_manifest_pending.is_none());
session.origin_account_binding.as_mut().unwrap().generation = 7;
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone()));
assert_eq!(session.origin_account_binding.as_ref().unwrap().generation, 7);
session.replace_origin_account_binding(Some(expected.clone()));
session.mark_for_gc_removal();
assert!(!publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement.clone()));
session.clear_gc_removal_mark();
assert!(publish_hls_origin_account_rebind(&mut session, &expected, 1, replacement));
assert_eq!(session.origin_account_binding.as_ref().unwrap().account_name.as_ref(), "replacement");
assert_eq!(session.activity.origin_work_generation, 2);
assert_eq!(session.origin_account_manifest_pending, Some(1));
}
}
@@ -0,0 +1,58 @@
use super::{AppState, HlsOriginIoContext, HlsSessionHandle};
use axum::http::HeaderMap;
use std::sync::Arc;
use tuliprox_hls::api::{
HlsAccessLeaseId, HlsManifestAcceptanceDirective, HlsManifestCommitRequirement, HlsPostRefreshRuntime,
HlsPreparedOriginAccount, OriginRefreshRequest, RetryPolicy,
};
pub(super) struct HlsOriginRefreshOptions {
pub headers: HeaderMap,
pub origin_io: HlsOriginIoContext,
pub manifest_commit_requirement: HlsManifestCommitRequirement,
pub acceptance_directive: HlsManifestAcceptanceDirective,
pub access_lease_id: Option<HlsAccessLeaseId>,
pub rewrite_secret: Vec<u8>,
pub now_ms: u64,
}
pub(super) fn hls_origin_refresh_request(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
prepared: HlsPreparedOriginAccount,
options: HlsOriginRefreshOptions,
) -> OriginRefreshRequest {
let mut origin_io = options.origin_io;
let origin_provider_session_headers = prepared.snapshot.provider_session_headers.clone();
origin_io.prepared_account = Some(prepared.snapshot);
OriginRefreshRequest {
app_config: Arc::clone(&app_state.app_config),
session: Arc::clone(session),
origin_entry: prepared.entry,
headers: options.headers,
origin_provider_session_headers,
disabled_headers: app_state.get_disabled_headers(),
client: app_state.http_clients.default.load().as_ref().clone(),
no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(),
use_manual_redirects: app_state.should_use_manual_redirects(),
segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()),
hls_proxy: Arc::clone(&app_state.hls.proxy),
segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()),
segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()),
map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()),
origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(),
manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(),
strip: app_state.hls.proxy.strip().clone(),
retry_policy: RetryPolicy::default(),
reverse_proxy_rewrite_secret: options.rewrite_secret,
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
manifest_commit_requirement: options.manifest_commit_requirement,
fresh_manifest_requirement_generation: None,
refresh_identity: None,
acceptance_directive: options.acceptance_directive,
access_lease_id: options.access_lease_id,
now_ms: options.now_ms,
origin_io: Some(origin_io),
post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: app_state.hls_ctx().downgrade() }),
}
}
@@ -186,7 +186,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_cached_segment(
match &lease.playback_mode {
HlsLeasePlaybackMode::Live => true,
HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq),
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false,
HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
| HlsLeasePlaybackMode::Ended => false,
}
}
@@ -94,7 +94,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_runtime_or_standalone_custom
return fallback_status.into_response();
};
match &lease.playback_mode {
HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
HlsLeasePlaybackMode::TerminalTail(_)
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) {
return response;
}
@@ -196,7 +198,9 @@ pub(in crate::api::endpoints::hls_api) fn hls_origin_runtime_resource_failure_re
err: HlsOriginRuntimeAcquireError,
) -> axum::response::Response {
match err {
HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(),
HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => {
hls_canonical_status_response(StatusCode::SERVICE_UNAVAILABLE)
}
HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status),
}
}
@@ -27,6 +27,10 @@ pub(in crate::api::endpoints::hls_api) async fn ensure_hls_cache_stream_register
access: &HlsAccessContext,
session: &HlsSessionHandle,
) -> Option<StreamInfo> {
// Share the existing origin-publication gate: a registration cannot retain an old
// account snapshot across a handover and then overwrite its projected provider.
let prepare_lock = Arc::clone(&session.read().await.origin_account_prepare_lock);
let _prepare_guard = prepare_lock.lock().await;
let (proxy_session_id, origin_source, origin_account_binding) = {
let session = session.read().await;
if session.is_gc_marked_for_removal() {
@@ -145,9 +145,9 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_resource(
match &lease.playback_mode {
HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await,
HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await,
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => {
StatusCode::NOT_FOUND.into_response()
}
HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
| HlsLeasePlaybackMode::Ended => StatusCode::NOT_FOUND.into_response(),
}
}
@@ -226,7 +226,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc
};
let resource = cache_resolution.resource;
let origin_headers = cache_resolution.origin_headers;
let origin_provider_session_headers = cache_resolution.origin_provider_session_headers;
let cache_action = cache_resolution.action;
match cache_action {
@@ -266,7 +265,6 @@ pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resourc
resource,
cache_action,
origin_headers,
origin_provider_session_headers,
cache_duration_ms,
})
.await
@@ -277,21 +275,14 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientPassthroughContext<'a>
pub(in crate::api::endpoints::hls_api) resource: TransientResourceRef,
pub(in crate::api::endpoints::hls_api) cache_action: HlsTransientObjectCacheAction,
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64,
}
pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_resource(
context: HlsTransientPassthroughContext<'_>,
) -> axum::response::Response {
let HlsTransientPassthroughContext {
endpoint,
resource,
cache_action,
origin_headers,
origin_provider_session_headers,
cache_duration_ms,
} = context;
let HlsTransientPassthroughContext { endpoint, resource, cache_action, origin_headers, cache_duration_ms } =
context;
if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action {
return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext {
app_state: endpoint.app_state,
@@ -304,7 +295,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r
resource_file: endpoint.resource_file,
fetch_token: *fetch_token,
origin_headers,
origin_provider_session_headers,
range_header: endpoint.range_header,
cache_duration_ms,
})
@@ -321,7 +311,6 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_r
resource: &resource,
resource_file: &endpoint.resource_file,
origin_headers,
origin_provider_session_headers,
range_header: endpoint.range_header.clone(),
policy: policy.clone(),
})
@@ -229,6 +229,7 @@ pub(in crate::api::endpoints::hls_api) async fn hls_proxy_terminal_segment(
}
HlsLeasePlaybackMode::Live
| HlsLeasePlaybackMode::TerminalTail(_)
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
| HlsLeasePlaybackMode::Ended => None,
},
@@ -318,6 +319,8 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context(
fingerprint: &Fingerprint,
preacquired_provider_handle: Option<ProviderHandle>,
) -> SegmentFetchContext {
let preacquired_provider_handle = preacquired_provider_handle
.map(|handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle));
let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = {
let session = session.read().await;
(
@@ -336,9 +339,12 @@ pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context(
reservation_ttl_secs,
preacquired_provider_handle: None,
started_generation: None,
prepared_account: None,
playback_kind: session.read().await.origin_playback_kind(),
account_runtime: session.read().await.origin_account_runtime.clone(),
};
if let Some(provider_handle) = preacquired_provider_handle {
origin_io = origin_io.with_preacquired_provider_handle(provider_handle);
origin_io = origin_io.with_managed_provider_handle(provider_handle);
}
SegmentFetchContext {
session: Arc::clone(session),
@@ -35,7 +35,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointOriginFetchReq
pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef,
pub(in crate::api::endpoints::hls_api) resource_file: &'a TransientResourceFile,
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) range_header: Option<HeaderValue>,
pub(in crate::api::endpoints::hls_api) policy: SegmentFetchPolicy,
}
@@ -54,14 +53,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_
no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(),
use_manual_redirects: request.app_state.should_use_manual_redirects(),
};
let log_identity = {
let (log_identity, generation, provider_headers) = {
let session = request.session.read().await;
HlsLogIdentity::from_session(&session)
(
HlsLogIdentity::from_session(&session),
session.activity.origin_work_generation,
session.origin_provider_session_headers.clone(),
)
};
let fetch_request = HlsTransientOriginFetchRequest {
resolved_origin_uri: request.resource.resolved_origin_uri.clone(),
origin_headers: request.origin_headers,
origin_provider_session_headers: request.origin_provider_session_headers,
origin_provider_session_headers: provider_headers,
range_header: request.range_header,
resource_file: request.resource_file.clone(),
resource_kind: request.resource.kind,
@@ -77,12 +80,19 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_
request.fingerprint,
request.headers,
&runtime_prepare_error,
HlsTransientOriginBindingSnapshot { resource: request.resource.clone(), generation },
);
let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await;
let runtime_prepare_error = *runtime_prepare_error.lock().await;
HlsTransientOriginFetchResult { result, runtime_prepare_error }
}
#[derive(Clone)]
pub(in crate::api::endpoints::hls_api) struct HlsTransientOriginBindingSnapshot {
resource: TransientResourceRef,
generation: u64,
}
/// Builds the shared per-attempt prepare closure for transient origin fetches.
/// Runtime acquire failures are captured in `runtime_prepare_error` and mapped
/// to a provider-unavailable fetch error so the retry loop can proceed uniformly.
@@ -93,6 +103,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
fingerprint: &Fingerprint,
headers: &HeaderMap,
runtime_prepare_error: &Arc<tokio::sync::Mutex<Option<HlsOriginRuntimeAcquireError>>>,
snapshot: HlsTransientOriginBindingSnapshot,
) -> impl FnMut(
HlsResourceFetchAttempt,
) -> futures::future::BoxFuture<
@@ -112,6 +123,7 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
let fingerprint = fingerprint.clone();
let headers = headers.clone();
let runtime_prepare_error = Arc::clone(&runtime_prepare_error);
let snapshot = snapshot.clone();
async move {
match prepare_hls_transient_origin_io_for_authorized_resource_work(
&app_state,
@@ -123,7 +135,17 @@ pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure(
)
.await
{
Ok(guard) => Ok(guard),
Ok(guard) => {
let session = session.read().await;
if session.activity.origin_work_generation != snapshot.generation
|| !session.transient.origin_resource_is_fetchable(&snapshot.resource, current_time_millis())
{
return Err(HlsOriginResourceFetchError::ProviderUnavailable(
HlsBoundAccountAcquireErrorKind::StoreRace,
));
}
Ok(guard)
}
Err(err) => {
*runtime_prepare_error.lock().await = Some(err);
Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable))
@@ -146,7 +168,6 @@ pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointCacheFetchCont
pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile,
pub(in crate::api::endpoints::hls_api) fetch_token: TransientObjectFetchToken,
pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap,
pub(in crate::api::endpoints::hls_api) range_header: Option<HeaderValue>,
pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64,
}
@@ -298,14 +319,18 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin
no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(),
use_manual_redirects: context.app_state.should_use_manual_redirects(),
};
let log_identity = {
let (log_identity, generation, provider_headers) = {
let session = context.session.read().await;
HlsLogIdentity::from_session(&session)
(
HlsLogIdentity::from_session(&session),
session.activity.origin_work_generation,
session.origin_provider_session_headers.clone(),
)
};
let fetch_request = HlsTransientOriginFetchRequest {
resolved_origin_uri: context.resource.resolved_origin_uri.clone(),
origin_headers: context.origin_headers.clone(),
origin_provider_session_headers: context.origin_provider_session_headers.clone(),
origin_provider_session_headers: provider_headers,
range_header: None,
resource_file: context.resource_file.clone(),
resource_kind: context.resource.kind,
@@ -336,6 +361,7 @@ pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin
context.fingerprint,
context.headers,
&runtime_prepare_error,
HlsTransientOriginBindingSnapshot { resource: context.resource.clone(), generation },
);
let final_failure =
match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt)
+130 -479
View File
@@ -378,13 +378,37 @@ pub(super) fn hls_url_failover_provider_for_origin_context(
}
pub(super) struct PreparedHlsOriginRuntime {
pub(super) fetch_url: String,
pub(super) runtime_provider_config: Option<Arc<RuntimeProviderConfig>>,
pub(super) url_failover_provider: Option<Arc<ConfigProvider>>,
pub(super) snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot,
pub(super) preacquired_origin_account_handle: Option<tuliprox_session::ManagedProviderHandle>,
}
impl PreparedHlsOriginRuntime {
pub(super) fn published(
allocation: HlsOriginRuntimeAllocation,
snapshot: tuliprox_hls::api::HlsOriginAccountSnapshot,
) -> Self {
Self {
fetch_url: allocation.fetch_url,
runtime_provider_config: allocation.runtime_provider_config,
url_failover_provider: allocation.url_failover_provider,
snapshot,
preacquired_origin_account_handle: allocation.preacquired_origin_account_handle,
}
}
}
pub(super) struct HlsOriginRuntimeAllocation {
pub(super) fetch_url: String,
pub(super) runtime_provider_config: Option<Arc<RuntimeProviderConfig>>,
// URL failover comes from source.yml provider:// resolution. Origin-account
// binding/handles are runtime account reservations and must stay separate.
pub(super) url_failover_provider: Option<Arc<ConfigProvider>>,
pub(super) origin_account_binding_to_store: Option<HlsOriginAccountBinding>,
pub(super) preacquired_origin_account_handle: Option<ProviderHandle>,
pub(super) binding_to_publish: Option<HlsOriginAccountBinding>,
pub(super) prepared_snapshot: Option<tuliprox_hls::api::HlsOriginAccountSnapshot>,
pub(super) preacquired_origin_account_handle: Option<tuliprox_session::ManagedProviderHandle>,
}
pub(super) fn effective_hls_url_failover_provider_for_fetch_url(
@@ -626,194 +650,8 @@ pub(super) fn hls_soft_overlap_delay_ms(
///
/// Archive-backed sessions must keep catchup semantics instead of plain live HLS;
/// live sessions keep the live HLS kind.
async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind {
if session.read().await.key.archive_reference.is_some() {
PlaybackKind::Catchup
} else {
PlaybackKind::LiveHls
}
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
pub(super) async fn prepare_hls_origin_runtime(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
proxy_session_id: &ProxySessionId,
fingerprint: &Fingerprint,
connection_kind: crate::api::model::ConnectionKind,
priority: i8,
work_kind: HlsOriginWorkKind,
work_class: HlsOriginWorkClass,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
let playback_kind = hls_session_origin_playback_kind(session).await;
promote_elapsed_hls_account_overlaps(app_state, now_ms).await;
detach_unprotected_hls_origin_account_bindings(app_state, now_ms).await;
reclaim_hls_account_overlap_if_needed(app_state, session, now_ms).await;
let existing_binding = session.read().await.origin_account_binding.clone();
let reacquire_detached_binding = existing_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached);
let final_no_account_reason = hls_no_account_reason_for_binding(existing_binding.as_ref());
if reacquire_detached_binding {
log_hls_origin_binding_reacquire_started(session, work_kind).await;
}
if let Some(binding) = existing_binding {
if binding.is_active() {
match hls_origin_account_status(&app_state.hls_ctx(), &binding) {
stale_status @ (HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired) => {
return rebind_hls_origin_account(
app_state,
session,
input,
raw_request_url,
session_entry_url,
&binding,
stale_status,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await;
}
HlsOriginAccountStatus::Known => {
return Ok(prepared_hls_origin_runtime_for_known_binding(
app_state,
input,
raw_request_url,
session_entry_url,
&binding,
)
.await);
}
}
}
}
match prepare_hls_origin_runtime_with_new_account(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
false,
work_kind,
work_class,
playback_kind,
now_ms,
)
.await
{
Ok(prepared) => {
if reacquire_detached_binding {
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => return Err(HlsOriginRuntimeAcquireError::Fatal(status)),
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
}
if work_class.allows_speculative_overlap() {
if let Ok(prepared) = prepare_hls_origin_policy_preempt_runtime(
app_state,
session,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await
{
if reacquire_detached_binding {
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
if let Ok(prepared) = prepare_hls_speculative_origin_runtime(
app_state,
session,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
playback_kind,
now_ms,
)
.await
{
if reacquire_detached_binding {
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
} else {
debug!("HLS account overlap skipped: work_class={} reason=background-origin-work", work_class.as_log_value());
}
if work_class.allows_grace() {
match prepare_hls_origin_runtime_with_new_account(
app_state,
input,
raw_request_url,
session_entry_url,
proxy_session_id,
fingerprint,
connection_kind,
priority,
true,
work_kind,
work_class,
playback_kind,
now_ms,
)
.await
{
Ok(prepared) => {
if reacquire_detached_binding {
if let Some(binding) = prepared.origin_account_binding_to_store.as_ref() {
log_hls_origin_binding_reacquired(session, binding).await;
}
}
return Ok(prepared);
}
Err(HlsOriginRuntimeAcquireError::Fatal(status)) => {
return Err(HlsOriginRuntimeAcquireError::Fatal(status))
}
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { .. }) => {}
}
} else {
debug!(
"HLS origin account grace skipped: work_class={} reason=background-origin-work",
work_class.as_log_value()
);
}
if reacquire_detached_binding {
log_hls_origin_binding_reacquire_failed(session, "no-account-available").await;
}
Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { reason: final_no_account_reason })
pub(super) async fn hls_session_origin_playback_kind(session: &HlsSessionHandle) -> PlaybackKind {
session.read().await.origin_playback_kind()
}
#[allow(clippy::too_many_arguments)]
@@ -831,7 +669,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
work_class: HlsOriginWorkClass,
playback_kind: PlaybackKind,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
let session_owner = build_hls_origin_session_owner(proxy_session_id);
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
&input.name,
@@ -852,8 +690,13 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
});
};
let managed_handle =
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
let Some(provider_handle) = managed_handle.handle() else {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
@@ -866,7 +709,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
)
.await
else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
@@ -876,7 +719,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
&provider_handle.allocation,
now_ms,
) else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
@@ -896,7 +739,7 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
grace_state
);
Ok(PreparedHlsOriginRuntime {
Ok(HlsOriginRuntimeAllocation {
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
raw_request_url,
@@ -905,8 +748,9 @@ pub(super) async fn prepare_hls_origin_runtime_with_new_account(
),
runtime_provider_config: Some(provider_config),
fetch_url,
origin_account_binding_to_store: Some(binding),
preacquired_origin_account_handle: Some(provider_handle),
binding_to_publish: Some(binding),
prepared_snapshot: None,
preacquired_origin_account_handle: Some(managed_handle),
})
}
@@ -916,7 +760,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
raw_request_url: &str,
session_entry_url: &str,
binding: &HlsOriginAccountBinding,
) -> PreparedHlsOriginRuntime {
) -> HlsOriginRuntimeAllocation {
let runtime_provider_config = app_state.active_provider.find_provider_config(&binding.account_name);
let fetch_url = resolve_hls_origin_fetch_url(
&app_state.app_config,
@@ -928,7 +772,7 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
.await
.unwrap_or_else(|| session_entry_url.to_string());
PreparedHlsOriginRuntime {
HlsOriginRuntimeAllocation {
runtime_provider_config,
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
@@ -937,7 +781,8 @@ pub(super) async fn prepared_hls_origin_runtime_for_known_binding(
&fetch_url,
),
fetch_url,
origin_account_binding_to_store: None,
binding_to_publish: None,
prepared_snapshot: None,
preacquired_origin_account_handle: None,
}
}
@@ -995,14 +840,15 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A
timing.target_duration_ms
);
let starting_window_ms = tuliprox_session::STARTING_LEASE_TTL_SECS.saturating_mul(1000);
let media_window_expired = session_guard.activity.last_delivered_media_at_ms.map_or_else(
|| now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms),
|last| {
now_ms
> last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms)
},
);
if !media_window_expired
// A retained session can contain delivery from an earlier playback.
// Give each newly published binding its own bounded start window;
// healthy preparation leaves pinned_at_ms unchanged.
let starting_window_expired = now_ms > binding.pinned_at_ms.saturating_add(starting_window_ms);
let media_window_expired = session_guard.activity.last_delivered_media_at_ms.is_none_or(|last| {
now_ms > last.saturating_add(timing.hard_active_window_ms).saturating_add(timing.soft_active_window_ms)
});
if !starting_window_expired
|| !media_window_expired
|| session_guard.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0
{
continue;
@@ -1033,7 +879,6 @@ pub(super) async fn detach_unprotected_hls_origin_account_bindings(app_state: &A
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
app_state: &Arc<AppState>,
new_session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
@@ -1043,7 +888,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
priority: i8,
playback_kind: PlaybackKind,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
let request_policy = HlsEffectiveOriginAcquirePolicy::new(connection_kind, priority, now_ms);
let Some(candidate) =
find_hls_origin_policy_preempt_candidate(app_state, input, proxy_session_id, request_policy, now_ms).await
@@ -1087,8 +932,13 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
debug!("HLS origin policy preemption denied: reason=exact-acquire-failed");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let managed_handle =
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
let Some(provider_handle) = managed_handle.handle() else {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS origin policy preemption denied: reason=missing-provider-config");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1102,7 +952,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
)
.await
else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS origin policy preemption denied: reason=invalid-origin-url");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1113,7 +963,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
&provider_handle.allocation,
now_ms,
) else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS origin policy preemption denied: reason=invalid-allocation");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1140,16 +990,12 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
}
}
if !detached_victim {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS origin policy preemption denied: reason=stale-candidate");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
{
let mut session_guard = new_session.write().await;
session_guard.replace_origin_account_binding(Some(binding.clone()));
}
debug!(
"HLS origin policy preempted: account={} victim_proxy_session={} winner_proxy_session={} victim_kind={:?} victim_priority={} request_kind={:?} request_priority={}",
sanitize_sensitive_info(candidate.account_name.as_ref()),
@@ -1167,7 +1013,7 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
HlsOriginAccountDetachedReason::PreemptedByHigherPriority.as_log_reason()
);
Ok(PreparedHlsOriginRuntime {
Ok(HlsOriginRuntimeAllocation {
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
raw_request_url,
@@ -1176,8 +1022,9 @@ pub(super) async fn prepare_hls_origin_policy_preempt_runtime(
),
runtime_provider_config: Some(provider_config),
fetch_url,
origin_account_binding_to_store: Some(binding),
preacquired_origin_account_handle: Some(provider_handle),
binding_to_publish: Some(binding),
prepared_snapshot: None,
preacquired_origin_account_handle: Some(managed_handle),
})
}
@@ -1283,7 +1130,6 @@ pub(super) fn hls_origin_policy_preempt_candidate_is_better(
#[allow(clippy::too_many_arguments)]
pub(super) async fn prepare_hls_speculative_origin_runtime(
app_state: &Arc<AppState>,
new_session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
@@ -1293,7 +1139,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
priority: i8,
playback_kind: PlaybackKind,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
) -> Result<HlsOriginRuntimeAllocation, HlsOriginRuntimeAcquireError> {
let Some(candidate) = find_hls_account_overlap_candidate(app_state, input, proxy_session_id, now_ms).await else {
debug!("HLS account overlap denied: reason=no-soft-active-candidate");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1316,8 +1162,13 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
debug!("HLS account overlap denied: reason=speculative-acquire-failed");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let managed_handle =
tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), provider_handle);
let Some(provider_handle) = managed_handle.handle() else {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS account overlap denied: reason=missing-provider-config");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1331,7 +1182,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
)
.await
else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
drop(managed_handle);
restore_hls_origin_policy_preempt_candidate_reservation(app_state, &candidate, playback_kind).await;
debug!("HLS account overlap denied: reason=invalid-origin-url");
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
@@ -1344,10 +1195,6 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
candidate.reclaim_until_ms,
now_ms,
);
{
let mut session_guard = new_session.write().await;
session_guard.replace_origin_account_binding(Some(binding.clone()));
}
debug!(
"HLS account overlap granted: account={} victim_proxy_session={} winner_proxy_session={} reclaim_until_ms={} eligible_after_ms={} delay_ms={} tuliprox_target_user_connections={} origin_input_account_connections={}",
sanitize_sensitive_info(candidate.account_name.as_ref()),
@@ -1359,7 +1206,7 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
candidate.tuliprox_target_user_connection_capacity,
candidate.origin_input_account_connection_capacity
);
Ok(PreparedHlsOriginRuntime {
Ok(HlsOriginRuntimeAllocation {
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
raw_request_url,
@@ -1368,8 +1215,9 @@ pub(super) async fn prepare_hls_speculative_origin_runtime(
),
runtime_provider_config: Some(provider_config),
fetch_url,
origin_account_binding_to_store: Some(binding),
preacquired_origin_account_handle: Some(provider_handle),
binding_to_publish: Some(binding),
prepared_snapshot: None,
preacquired_origin_account_handle: Some(managed_handle),
})
}
@@ -1650,154 +1498,6 @@ pub(super) async fn promote_elapsed_hls_account_overlaps(app_state: &Arc<AppStat
}
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_lines)]
pub(super) async fn rebind_hls_origin_account(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
input: &ConfigInput,
raw_request_url: &str,
session_entry_url: &str,
stale_binding: &HlsOriginAccountBinding,
stale_status: HlsOriginAccountStatus,
fingerprint: &Fingerprint,
connection_kind: crate::api::model::ConnectionKind,
priority: i8,
playback_kind: PlaybackKind,
now_ms: u64,
) -> Result<PreparedHlsOriginRuntime, HlsOriginRuntimeAcquireError> {
{
let mut session_guard = session.write().await;
if !session_guard.origin_account_rebind.is_allowed_now(now_ms) {
debug!(
"HLS origin account rebind skipped by backoff: proxy_session={} old_account={} retry_after_ms=2000",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
session_guard.origin_account_rebind.mark_attempt_started(Arc::clone(&stale_binding.account_name), now_ms);
}
let safe_proxy_session = {
let session_guard = session.read().await;
safe_proxy_session_id(&session_guard.proxy_session_id)
};
debug!(
"HLS origin account rebind started: proxy_session={} old_account={} reason={stale_status:?}",
safe_proxy_session,
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
app_state.active_provider.clear_identified_provider_reservation(
&stale_binding.session_owner,
&stale_binding.account_name,
stale_binding.provider_binding_tag,
);
{
let mut session_guard = session.write().await;
if let Some(binding) = session_guard.origin_account_binding.as_mut().filter(|binding| {
binding.account_name == stale_binding.account_name
&& binding.session_owner == stale_binding.session_owner
&& binding.provider_binding_tag == stale_binding.provider_binding_tag
}) {
binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, now_ms);
debug!(
"HLS origin binding detached: proxy_session={} account={} reason={}",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
HlsOriginAccountDetachedReason::AccountMissingOrExpired.as_log_reason()
);
}
session_guard.invalidate_queued_origin_work();
}
let Some(provider_handle) = app_state.active_provider.acquire_connection_with_lease_for_session(
&input.name,
&fingerprint.addr,
false,
priority,
connection_kind,
Some(PlaybackLeaseRef::new(&stale_binding.session_owner, playback_kind)),
) else {
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_account_available").await;
return Err(HlsOriginRuntimeAcquireError::NoAccountAvailable {
reason: hls_no_account_reason_for_binding(Some(stale_binding)),
});
};
let Some(provider_config) = provider_handle.allocation.get_provider_config() else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "no_provider_config").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(fetch_url) = resolve_hls_origin_fetch_url(
&app_state.app_config,
input,
raw_request_url,
session_entry_url,
Some(&provider_config),
)
.await
else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "invalid_origin_url").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let Some(new_account_name) = provider_handle.allocation.get_provider_name() else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
mark_hls_origin_rebind_failed(session, stale_binding, now_ms, "missing_account_name").await;
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let new_binding = HlsOriginAccountBinding::rebound(
Arc::clone(&input.name),
new_account_name,
stale_binding.session_owner.clone(),
stale_binding.generation.saturating_add(1),
now_ms,
);
{
let mut session_guard = session.write().await;
session_guard.replace_origin_account_binding(Some(new_binding.clone()));
session_guard.origin_account_rebind.mark_success();
}
debug!(
"HLS origin account rebound: old_account={} new_account={}",
sanitize_sensitive_info(stale_binding.account_name.as_ref()),
sanitize_sensitive_info(new_binding.account_name.as_ref())
);
Ok(PreparedHlsOriginRuntime {
url_failover_provider: hls_url_failover_provider_for_origin_context(
input,
raw_request_url,
session_entry_url,
&fetch_url,
),
runtime_provider_config: Some(provider_config),
fetch_url,
origin_account_binding_to_store: None,
preacquired_origin_account_handle: Some(provider_handle),
})
}
pub(super) async fn mark_hls_origin_rebind_failed(
session: &HlsSessionHandle,
stale_binding: &HlsOriginAccountBinding,
now_ms: u64,
reason: &str,
) {
let mut session_guard = session.write().await;
session_guard.origin_account_rebind.mark_failed(now_ms);
debug!(
"HLS origin account rebind failed: proxy_session={} old_account={} reason={reason} retry_after_ms=2000",
safe_proxy_session_id(&session_guard.proxy_session_id),
sanitize_sensitive_info(stale_binding.account_name.as_ref())
);
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn prepare_hls_cache_user_session(
app_state: &Arc<AppState>,
@@ -1995,6 +1695,7 @@ pub(super) async fn hls_segment_request_requires_origin_work(
&& entry.origin_fetch_ref.is_some()
}
#[cfg(test)]
pub(super) async fn hls_origin_binding_needs_reacquire(session: &HlsSessionHandle) -> bool {
let session = session.read().await;
session.origin_account_binding.as_ref().is_some_and(HlsOriginAccountBinding::is_detached)
@@ -2008,7 +1709,7 @@ pub(super) fn hls_transient_origin_binding_requires_runtime_prepare(
|| (binding.is_active()
&& matches!(
hls_origin_account_status(hls_ctx, binding),
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired | HlsOriginAccountStatus::Blocked
))
}
@@ -2021,11 +1722,22 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work(
work_kind: HlsOriginWorkKind,
now_ms: u64,
) -> Result<Option<ProviderHandle>, HlsOriginRuntimeAcquireError> {
if !hls_origin_binding_needs_reacquire(session).await {
let runtime = session.read().await.origin_account_runtime.clone();
if let Some(runtime) = runtime {
runtime
.prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand)
.await
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
return Ok(None);
}
if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
if !session
.read()
.await
.origin_account_binding
.as_ref()
.is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&app_state.hls_ctx(), binding))
{
return Ok(None);
}
let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers)
.await
@@ -2047,13 +1759,17 @@ pub(super) async fn prepare_hls_origin_binding_for_authorized_resource_work(
now_ms,
)
.await?;
if let Some(binding) = prepared_origin.origin_account_binding_to_store {
session.write().await.replace_origin_account_binding(Some(binding));
drop(prepared_origin);
let runtime = session.read().await.origin_account_runtime.clone();
if let Some(runtime) = runtime {
runtime
.prepare(session, tuliprox_hls::api::HlsOriginAccountWork::Media, HlsOriginWorkClass::Demand)
.await
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
}
Ok(prepared_origin.preacquired_origin_account_handle)
Ok(None)
}
#[allow(clippy::too_many_lines)]
pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work(
app_state: &Arc<AppState>,
session: &HlsSessionHandle,
@@ -2062,120 +1778,55 @@ pub(super) async fn prepare_hls_transient_origin_io_for_authorized_resource_work
req_headers: &HeaderMap,
now_ms: u64,
) -> Result<Option<HlsTransientOriginIoGuard>, HlsOriginRuntimeAcquireError> {
let hls_ctx = app_state.hls_ctx();
let existing_binding = session.read().await.origin_account_binding.clone();
let origin_policy = hls_effective_origin_acquire_policy(session).await;
let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(session).await;
if let Some(binding) = existing_binding.as_ref().filter(|binding| binding.is_active()) {
match hls_origin_account_status(&hls_ctx, binding) {
HlsOriginAccountStatus::Known => {
let origin_io = HlsOriginIoContext {
ctx: hls_ctx.clone(),
client_addr: fingerprint.addr,
allow_grace: HlsOriginWorkClass::Demand.allows_grace(),
priority: origin_policy.priority,
connection_kind: origin_policy.connection_kind,
reservation_ttl_secs,
preacquired_provider_handle: None,
started_generation: None,
};
if let Ok(lease_guard) = begin_hls_origin_account_io_bounded(
&origin_io,
session,
binding,
hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms),
)
.await
{
let (started_generation, active_origin_work_count) = {
let mut session = session.write().await;
let count = Arc::clone(&session.activity.active_origin_work_count);
(session.start_origin_work(), count)
};
return Ok(Some(HlsTransientOriginIoGuard::new(
Arc::clone(session),
active_origin_work_count,
origin_io,
lease_guard,
started_generation,
)));
}
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
HlsOriginAccountStatus::Missing | HlsOriginAccountStatus::Expired => {}
}
}
if !existing_binding
.as_ref()
.is_some_and(|binding| hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, binding))
{
return Ok(None);
}
if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) > 0 {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
let request_context = resolve_hls_playback_manifest_request_context(app_state, access_context, req_headers)
.await
.map_err(HlsOriginRuntimeAcquireError::Fatal)?;
let proxy_session_id = session.read().await.proxy_session_id.clone();
let prepared_origin = prepare_hls_origin_runtime(
prepare_hls_origin_binding_for_authorized_resource_work(
app_state,
session,
&request_context.input,
&request_context.hls_url,
request_context.session_entry_url.as_str(),
&proxy_session_id,
access_context,
fingerprint,
origin_policy.connection_kind,
origin_policy.priority,
req_headers,
HlsOriginWorkKind::Resource,
HlsOriginWorkClass::Demand,
now_ms,
)
.await?;
if let Some(binding) = prepared_origin.origin_account_binding_to_store {
session.write().await.replace_origin_account_binding(Some(binding));
}
let Some(provider_handle) = prepared_origin.preacquired_origin_account_handle else {
let Some(binding) = session.read().await.origin_account_binding.clone() else {
return Ok(None);
};
let Some(binding) = session.read().await.origin_account_binding.clone().filter(HlsOriginAccountBinding::is_active)
else {
app_state.connection_manager.release_provider_handle(Some(provider_handle));
if binding.is_detached() {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
}
let policy = hls_effective_origin_acquire_policy(session).await;
let origin_io = HlsOriginIoContext {
ctx: hls_ctx,
ctx: app_state.hls_ctx(),
client_addr: fingerprint.addr,
allow_grace: HlsOriginWorkClass::Demand.allows_grace(),
priority: origin_policy.priority,
connection_kind: origin_policy.connection_kind,
reservation_ttl_secs,
priority: policy.priority,
connection_kind: policy.connection_kind,
reservation_ttl_secs: hls_origin_account_reservation_ttl_secs_for_session(session).await,
preacquired_provider_handle: None,
started_generation: None,
}
.with_preacquired_provider_handle(provider_handle);
let Ok(lease_guard) = begin_hls_origin_account_io_bounded(
prepared_account: None,
playback_kind: session.read().await.origin_playback_kind(),
account_runtime: session.read().await.origin_account_runtime.clone(),
};
let lease_guard = begin_hls_origin_account_io_bounded(
&origin_io,
session,
&binding,
hls_object_body_deadline(app_state.hls.proxy.segment_fetch_policy().origin_segment_timeout_ms),
)
.await
else {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
};
let (started_generation, active_origin_work_count) = {
.map_err(|_| HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE))?;
let (generation, active_count) = {
let mut session = session.write().await;
if !session.origin_account_binding.as_ref().is_some_and(|current| {
current.generation == binding.generation
&& current.account_name == binding.account_name
&& current.is_active()
}) {
return Err(HlsOriginRuntimeAcquireError::Fatal(StatusCode::SERVICE_UNAVAILABLE));
}
let count = Arc::clone(&session.activity.active_origin_work_count);
(session.start_origin_work(), count)
};
Ok(Some(HlsTransientOriginIoGuard::new(
Arc::clone(session),
active_origin_work_count,
origin_io,
lease_guard,
started_generation,
)))
Ok(Some(HlsTransientOriginIoGuard::new(Arc::clone(session), active_count, origin_io, lease_guard, generation)))
}
@@ -78,15 +78,20 @@ use tuliprox_hls::api::{
HlsCachedManifestOptions, HlsLeaseManifestSnapshotInput, LiveHlsOriginEntry,
};
mod origin_account_rebind;
mod owner_token;
mod reduced_retry;
mod same_account_resume;
mod archive;
mod bandwidth;
mod encrypted_media;
mod entry_identity;
mod headers;
mod late_custom_response;
mod lease_lifecycle;
mod manifest_entry_freshness;
mod manifest_short_window;
mod manifest_startup;
mod observability;
mod owner_handoff;
@@ -163,7 +163,7 @@ async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_sessi
.is_some());
assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none());
assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0);
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -398,7 +398,7 @@ async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_h
assert!(!response.headers().contains_key(header::LOCATION));
assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl");
assert!(app_state.hls.proxy.access_leases().read().await.is_empty());
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -424,7 +424,7 @@ async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_targ
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert!(app_state.hls.proxy.access_leases().read().await.is_empty());
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -545,7 +545,7 @@ async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_c
assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8");
assert!(reservation.selected_provider_config.is_some());
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
app_state.connection_manager.release_provider_handle(reservation.provider_handle);
// A manifest-only entry reservation is still unconfirmed, so it must not block a
@@ -1053,7 +1053,7 @@ async fn hls_entry_for_input_in_maintenance_opens_no_session() {
// No maintenance clip is configured here, so the configured fallback status answers.
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(app_state.hls.proxy.access_leases().read().await.is_empty(), "no lease for a maintenance input");
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -0,0 +1,259 @@
use super::*;
#[tokio::test]
async fn late_custom_response_serves_canonical_manifest_and_immutable_media_without_origin_requests() {
let fixture = terminal_tail::publication_late_fixture().await;
let now_ms = super::super::current_time_millis();
let lease = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("published live lease");
let snapshot = lease.last_manifest_snapshot.expect("published snapshot");
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let requests_before = fixture.origin.manifest_request_count();
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert!(!response.headers().contains_key(header::LOCATION));
assert!(!response.headers().contains_key(header::RETRY_AFTER));
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
assert!(body.contains(&format!("#EXT-X-MEDIA-SEQUENCE:{}\n", snapshot.last_proxy_seq + 1)));
assert!(body.contains(&format!("#EXT-X-TARGETDURATION:{}\n", snapshot.target_duration_ms / 1_000)));
assert!(body.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n"));
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
assert!(!body.contains("/hls/shared/live/"));
let media_uris = body.lines().filter(|line| line.contains("/cvs/hls/")).collect::<Vec<_>>();
assert_eq!(media_uris.len(), 12);
let (first, second) = tokio::join!(
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None),
);
for response in [first, second] {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response_body(response).await, body.as_bytes());
}
let media = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
assert_eq!(media.status(), StatusCode::OK);
assert_eq!(media.headers()[header::CONTENT_TYPE], "video/mp2t");
let bytes = response_body(media).await;
assert!(!bytes.is_empty());
let range = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], Some("bytes=0-187")).await;
assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT);
assert_eq!(response_body(range).await, bytes.slice(..188));
let router =
crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(Arc::clone(&fixture.app_state));
let head = router
.oneshot(
Request::builder()
.method(Method::HEAD)
.uri(media_uris[0])
.extension(ConnectInfo(test_addr()))
.body(Body::empty())
.expect("HEAD request"),
)
.await
.expect("HEAD response");
assert_eq!(head.status(), StatusCode::OK);
assert_eq!(head.headers()[header::CONTENT_LENGTH], bytes.len().to_string());
assert!(response_body(head).await.is_empty());
fixture.app_state.app_config.custom_stream_response.store(None);
let replay = custom_media_response(Arc::clone(&fixture.app_state), media_uris[0], None).await;
assert_eq!(replay.status(), StatusCode::OK);
assert_eq!(response_body(replay).await, bytes);
let wrong_lease = media_uris[0].replace(&fixture.access_lease_id.0, "unknown-lease");
assert_eq!(
custom_media_response(Arc::clone(&fixture.app_state), &wrong_lease, None).await.status(),
StatusCode::NOT_FOUND
);
let wrong_index = media_uris[0].replace("/0.ts", "/12.ts");
assert_eq!(
custom_media_response(Arc::clone(&fixture.app_state), &wrong_index, None).await.status(),
StatusCode::NOT_FOUND
);
let late = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.expect("terminal lease");
assert!(matches!(late.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
assert_eq!(late.playback_cursor, lease.playback_cursor);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
assert_eq!(fixture.origin.manifest_request_count(), requests_before);
}
#[tokio::test]
async fn late_custom_response_without_configured_asset_retains_error_response() {
let fixture = terminal_tail::publication_late_fixture().await;
fixture.app_state.app_config.custom_stream_response.store(None);
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(response_body(response).await.is_empty());
let lease = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.expect("terminal unavailable lease");
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. }));
}
#[tokio::test]
async fn late_custom_media_get_head_and_range_renew_lease_without_manifest_polls() {
let (fixture, media_uri) = committed_late_http_fixture().await;
for (method, range, status) in [
(Method::GET, None, StatusCode::OK),
(Method::HEAD, None, StatusCode::OK),
(Method::GET, Some("bytes=0-187"), StatusCode::PARTIAL_CONTENT),
] {
let now_ms = super::super::current_time_millis();
let previous_access_ms = now_ms.saturating_sub(50_000);
assert!(fixture
.app_state
.hls
.proxy
.activate_access_lease(
&fixture.access_lease_id,
&fixture.proxy_session_id,
previous_access_ms,
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let before = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("valid late lease");
let response =
custom_media_response_with_method(Arc::clone(&fixture.app_state), &media_uri, range, method).await;
assert_eq!(response.status(), status);
let after = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.expect("renewed late lease");
assert!(after.last_seen_at_ms >= now_ms);
assert!(after.valid_until_ms > before.valid_until_ms);
assert!(after.active_until_ms > before.active_until_ms);
assert_eq!(after.playback_mode, before.playback_mode);
assert_eq!(after.playback_cursor, before.playback_cursor);
assert_eq!(after.admission_generation, before.admission_generation);
}
}
#[tokio::test]
async fn late_custom_media_errors_and_denied_authorization_do_not_renew_lease() {
let (fixture, media_uri) = committed_late_http_fixture().await;
let now_ms = super::super::current_time_millis();
assert!(fixture
.app_state
.hls
.proxy
.activate_access_lease(
&fixture.access_lease_id,
&fixture.proxy_session_id,
now_ms - 50_000,
HlsAccessLeaseTiming { active_window_ms: 24_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let before = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(&fixture.access_lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("valid late lease");
let fingerprint = media_uri.rsplit('/').nth(1).expect("asset fingerprint");
for (uri, range, expected) in [
(media_uri.replace("/0.ts", "/12.ts"), None, StatusCode::NOT_FOUND),
(media_uri.replace(fingerprint, "wrong-fingerprint"), None, StatusCode::NOT_FOUND),
(media_uri.replace(&fixture.access_lease_id.0, "unknown-lease"), None, StatusCode::NOT_FOUND),
(media_uri.clone(), Some("bytes=1000000000000-"), StatusCode::RANGE_NOT_SATISFIABLE),
] {
let response = custom_media_response(Arc::clone(&fixture.app_state), &uri, range).await;
assert_eq!(response.status(), expected);
}
fixture.app_state.app_config.api_proxy.store(None);
let response = custom_media_response(Arc::clone(&fixture.app_state), &media_uri, None).await;
assert_eq!(response.status(), fixture.app_state.app_config.get_auth_error_status());
let after = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&fixture.access_lease_id,
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.expect("unchanged late lease");
assert_eq!(after.last_seen_at_ms, before.last_seen_at_ms);
assert_eq!(after.valid_until_ms, before.valid_until_ms);
assert_eq!(after.active_until_ms, before.active_until_ms);
}
async fn committed_late_http_fixture() -> (terminal_tail::PublicationLateFixture, String) {
let fixture = terminal_tail::publication_late_fixture().await;
{
let mut session = fixture.session.write().await;
session.segments.clear();
session.origin_control.path_condition = crate::api::model::HlsOriginPathCondition::AcceptanceConflict;
}
let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late manifest");
let media_uri = body.lines().find(|line| line.contains("/cvs/hls/")).expect("custom segment URI").to_owned();
(fixture, media_uri)
}
async fn custom_media_response(app_state: Arc<AppState>, uri: &str, range: Option<&str>) -> Response<Body> {
custom_media_response_with_method(app_state, uri, range, Method::GET).await
}
async fn custom_media_response_with_method(
app_state: Arc<AppState>,
uri: &str,
range: Option<&str>,
method: Method,
) -> Response<Body> {
let router = crate::api::endpoints::custom_video_stream_api::cvs_api_register().with_state(app_state);
let mut request = Request::builder().method(method).uri(uri);
if let Some(range) = range {
request = request.header(header::RANGE, range);
}
router
.oneshot(request.extension(ConnectInfo(test_addr())).body(Body::empty()).expect("custom media request"))
.await
.expect("custom media response")
}
@@ -17,11 +17,14 @@ use std::sync::{
const ORIGIN_WINDOW_SEGMENTS: u64 = 6;
const SEGMENT_BODY: &[u8] = &[0x47; 8_192];
fn origin_playlist(head: u64) -> Vec<u8> {
fn origin_playlist(head: u64) -> Vec<u8> { origin_playlist_with_duration(head, 2) }
pub(super) fn origin_playlist_with_duration(head: u64, target_duration: u64) -> Vec<u8> {
use std::fmt::Write;
let mut body = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
let mut body =
format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{target_duration}\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
for sequence in head..head + ORIGIN_WINDOW_SEGMENTS {
let _ = write!(body, "#EXTINF:2.0,\nseg-{sequence}.ts\n");
let _ = write!(body, "#EXTINF:{target_duration}.0,\nseg-{sequence}.ts\n");
}
body.into_bytes()
}
@@ -61,6 +64,20 @@ async fn first_origin_sequence_for_new_lease(
request_url: &str,
lease: &str,
) -> u64 {
let body = open_manifest_for_new_lease(app_state, input, request_url, lease).await;
let session_key = super::super::build_hls_origin_source(input, "12345").session_key();
let first_proxy_seq = manifest_media_sequence(&body);
let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session");
let session = session.read().await;
session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence
}
pub(super) async fn open_manifest_for_new_lease(
app_state: &Arc<AppState>,
input: &ConfigInput,
request_url: &str,
lease: &str,
) -> String {
let origin_source = super::super::build_hls_origin_source(input, "12345");
let session_key = origin_source.session_key();
let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret());
@@ -88,11 +105,7 @@ async fn first_origin_sequence_for_new_lease(
.await
.expect("hls cache should handle the live entrypoint");
assert_eq!(response.status(), StatusCode::OK, "lease {lease}");
let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8");
let first_proxy_seq = manifest_media_sequence(&body);
let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("shared hls session");
let session = session.read().await;
session.segments.get(&first_proxy_seq).expect("published segment is mapped").origin_key.host_local_sequence
String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8")
}
/// Moves the retained session's last origin confirmation `idle_ms` into the
@@ -0,0 +1,526 @@
use super::{support::TestEncodedManifestOrigin, *};
use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode};
use std::sync::atomic::AtomicU64;
use tuliprox_hls::api::HlsFreshManifestRequiredReason;
struct ShortWindowFixture {
_temp_dir: tempfile::TempDir,
origin: TestSegmentOrigin,
app_state: Arc<AppState>,
input: ConfigInput,
request_url: String,
head: Arc<AtomicU64>,
count: Arc<AtomicUsize>,
proxy_session_id: ProxySessionId,
}
enum ShortWindowResources {
Plain,
EncryptedWithMap,
FirstReady { media_base_url: String },
}
async fn short_window_fixture(count: usize) -> ShortWindowFixture {
short_window_resource_fixture(count, ShortWindowResources::Plain).await
}
async fn short_window_resource_fixture(count: usize, resources: ShortWindowResources) -> ShortWindowFixture {
let startup_mode = match &resources {
ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => HlsStartupMode::Conservative,
ShortWindowResources::FirstReady { .. } => HlsStartupMode::FirstReady,
};
let head = Arc::new(AtomicU64::new(100));
let count = Arc::new(AtomicUsize::new(count));
let (origin_head, origin_count) = (Arc::clone(&head), Arc::clone(&count));
let origin = spawn_test_binary_origin(Arc::new(move |path| {
let body = if path_has_extension(path, "m3u8") {
let head = origin_head.load(Ordering::SeqCst);
let mut body =
format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{head}\n");
let extension = match &resources {
ShortWindowResources::Plain | ShortWindowResources::FirstReady { .. } => "ts",
ShortWindowResources::EncryptedWithMap => {
body.push_str("#EXT-X-MAP:URI=\"init.mp4\"\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x01010101010101010101010101010101\n");
"m4s"
}
};
for offset in 0..origin_count.load(Ordering::SeqCst) {
let segment_url = match &resources {
ShortWindowResources::FirstReady { media_base_url } => {
format!("{media_base_url}/segment-{}.ts", head + u64::try_from(offset).unwrap())
}
ShortWindowResources::Plain | ShortWindowResources::EncryptedWithMap => {
format!("segment-{}.{extension}", head + u64::try_from(offset).unwrap())
}
};
writeln!(body, "#EXTINF:2.5,\n{segment_url}").unwrap();
}
Arc::from(body.into_bytes())
} else if path_has_extension(path, "mp4") {
Arc::from(&b"short-window-map"[..])
} else if path_has_extension(path, "bin") {
Arc::from(AES_TEST_KEY_BYTES)
} else {
match &resources {
ShortWindowResources::Plain => Arc::from(&b"short-window-media"[..]),
ShortWindowResources::FirstReady { .. } => unreachable!("media uses the gated segment origin"),
ShortWindowResources::EncryptedWithMap => Arc::from(encrypt_test_aes128_cbc_pkcs7(
b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])),
}
};
TestBinaryOriginResponse::new(StatusCode::OK, body)
}))
.await;
let temp_dir = tempfile::tempdir().unwrap();
let input = ConfigInput {
id: 1,
name: Arc::from("short-window-input"),
input_type: InputType::M3u,
url: origin.base_url.clone(),
max_connections: 1,
enabled: true,
..ConfigInput::default()
};
let app_state = test_app_state_with_hls_proxy_and_inputs(
Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)),
vec![Arc::new(input.clone())],
);
let mut cache = HlsCacheConfigDto {
cache_path: Some(temp_dir.path().to_string_lossy().to_string()),
startup: HlsStartupConfigDto { mode: startup_mode, ..HlsStartupConfigDto::default() },
..HlsCacheConfigDto::default()
};
cache.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Friendly;
let config = Config {
reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto {
hls_cache: Some(cache),
..Default::default()
})),
..Config::default()
};
app_state.app_config.config.store(Arc::new(config));
app_state.hls.proxy.update_config(&app_state.app_config).await;
configure_default_test_server(&app_state);
let mut target = test_m3u_hls_share_target();
target.name = "default".to_string();
store_test_sources_with_target(&app_state, input.clone(), target.clone());
create_active_hls_user_session(&app_state).await;
let request_url = format!("{}/channel/index.m3u8", origin.base_url);
cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "short-channel", &request_url)).await;
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
let proxy_session_id = crate::api::model::build_proxy_session_id(&key, &app_state.get_encrypt_secret());
ShortWindowFixture { _temp_dir: temp_dir, origin, app_state, input, request_url, head, count, proxy_session_id }
}
/// Keeps the first media fetch pending until all lease entries have been checked.
async fn gated_segment_origin(gate: Arc<tokio::sync::Semaphore>) -> TestEncodedManifestOrigin {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let requests = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let origin_requests = Arc::clone(&requests);
let task = tokio::spawn(async move {
while let Ok((mut socket, _)) = listener.accept().await {
let requests = Arc::clone(&origin_requests);
let gate = Arc::clone(&gate);
tokio::spawn(async move {
let mut buffer = [0_u8; 2048];
let read = socket.read(&mut buffer).await.unwrap();
requests.lock().await.push(String::from_utf8_lossy(&buffer[..read]).to_string());
let _permit = gate.acquire().await;
let body = [0x47; 8192];
let headers = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
socket.write_all(headers.as_bytes()).await.unwrap();
socket.write_all(&body).await.unwrap();
});
}
});
TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task }
}
async fn wait_for_shared_first_ready_refresh(
fixture: &ShortWindowFixture,
session: &HlsSessionHandle,
previous_requests: usize,
) {
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if fixture.origin.manifest_request_count() > previous_requests
&& !session.read().await.origin_refresh.in_flight
{
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("additional leases complete one shared freshness request");
assert_eq!(fixture.origin.manifest_request_count(), previous_requests + 1);
}
async fn assert_first_ready_lease_media(
fixture: &ShortWindowFixture,
leases: &mut tokio::task::JoinSet<String>,
minimum_media_sequence: u64,
) {
tokio::time::timeout(Duration::from_secs(5), async {
while let Some(result) = leases.join_next().await {
let body = result.unwrap();
assert!(manifest_media_sequence(&body) >= minimum_media_sequence);
assert_eq!(media_uri_count(&body), 1);
for uri in body.lines().filter(|line| !line.starts_with('#') && !line.is_empty()) {
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response_body(response).await.as_ref(), &[0x47; 8192]);
}
}
})
.await
.expect("media publication releases every waiting lease");
}
#[tokio::test]
async fn first_ready_leases_share_qualification_while_first_segment_is_loading() {
let gate = Arc::new(tokio::sync::Semaphore::new(0));
let media = gated_segment_origin(Arc::clone(&gate)).await;
let fixture = Arc::new(
short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() })
.await,
);
let mut leases = tokio::task::JoinSet::new();
let first = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&first, "first-ready-0").await });
let session = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if !media.requests.lock().await.is_empty() {
if let Some(session) =
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await
{
if !session.read().await.origin_refresh.in_flight {
break session;
}
}
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("qualified origin has started its first segment fetch");
let generation = {
let session = session.read().await;
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
assert!(session.last_rendered_manifest.is_none());
assert_eq!(session.origin_control.acceptance_generation.0, 1);
session.origin_control.acceptance_generation
};
assert_eq!(fixture.origin.manifest_request_count(), 3);
for lease in 1..20 {
let fixture = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&fixture, &format!("first-ready-{lease}")).await });
}
wait_for_shared_first_ready_refresh(&fixture, &session, 3).await;
assert!(session.read().await.last_rendered_manifest.is_none());
assert_eq!(session.read().await.origin_control.acceptance_generation, generation);
gate.close();
assert_first_ready_lease_media(&fixture, &mut leases, 0).await;
assert_eq!(session.read().await.origin_control.acceptance_generation, generation);
}
#[tokio::test]
async fn first_ready_revalidation_keeps_its_qualification_while_new_media_is_loading() {
let gate = Arc::new(tokio::sync::Semaphore::new(1));
let media = gated_segment_origin(Arc::clone(&gate)).await;
let fixture = Arc::new(
short_window_resource_fixture(1, ShortWindowResources::FirstReady { media_base_url: media.base_url.clone() })
.await,
);
let previous_body = open_short_lease(&fixture, "revalidation-baseline").await;
let session =
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
let previous_commit = {
let mut session = session.write().await;
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
assert_eq!(session.origin_control.acceptance_generation.0, 1);
session.last_rendered_manifest.as_mut().unwrap().valid_until_ms = 0;
session.activity.last_authorized_media_at_ms = None;
session.startup.as_mut().unwrap().revisions.clear();
for segment in session.segments.values_mut() {
segment.status = SegmentCacheStatus::Expired;
}
session.last_normal_manifest_commit_identity().unwrap()
};
assert_eq!(fixture.origin.manifest_request_count(), 3);
assert_eq!(gate.forget_permits(1), 1);
assert_eq!(
hls_manifest_commit_requirement(
&session,
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
None,
true,
super::super::current_time_millis(),
)
.await,
HlsManifestCommitRequirement::FreshCommitRequired {
reason: HlsFreshManifestRequiredReason::ExpiredRevalidation,
}
);
fixture.head.store(101, Ordering::SeqCst);
let mut leases = tokio::task::JoinSet::new();
let first = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&first, "revalidation-waiting-0").await });
let revalidation = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if media.requests.lock().await.len() >= 2 {
let state = session.read().await;
if !state.origin_refresh.in_flight && state.origin_seq_highwater == Some(101) {
break;
}
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await;
let media_requests = media.requests.lock().await.len();
{
let session = session.read().await;
assert!(
revalidation.is_ok(),
"revalidation requests={} generation={} highwater={:?} media_requests={media_requests} in_flight={} commit={:?}",
fixture.origin.manifest_request_count(),
session.origin_control.acceptance_generation.0,
session.origin_seq_highwater,
session.origin_refresh.in_flight,
session.last_normal_manifest_commit_identity(),
);
}
assert_eq!(fixture.origin.manifest_request_count(), 5);
{
let session = session.read().await;
assert_eq!(session.startup_mode(), HlsStartupMode::FirstReady);
assert_eq!(session.origin_control.acceptance_generation.0, 2);
assert_eq!(session.last_normal_manifest_commit_identity(), Some(previous_commit));
}
assert_eq!(
hls_manifest_commit_requirement(
&session,
tuliprox_hls::api::HlsSessionStoreOutcome::Reused,
None,
true,
super::super::current_time_millis(),
)
.await,
HlsManifestCommitRequirement::FreshCommitRequired {
reason: HlsFreshManifestRequiredReason::ExpiredRevalidation,
}
);
for lease in 1..20 {
let fixture = Arc::clone(&fixture);
leases.spawn(async move { open_short_lease(&fixture, &format!("revalidation-waiting-{lease}")).await });
}
wait_for_shared_first_ready_refresh(&fixture, &session, 5).await;
assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2);
assert_eq!(session.read().await.last_normal_manifest_commit_identity(), Some(previous_commit));
gate.close();
assert_first_ready_lease_media(&fixture, &mut leases, manifest_media_sequence(&previous_body) + 1).await;
assert_eq!(session.read().await.origin_control.acceptance_generation.0, 2);
assert!(session.read().await.last_normal_manifest_commit_identity().unwrap() > previous_commit);
}
async fn open_short_lease(fixture: &ShortWindowFixture, lease: &str) -> String {
let access_lease_id = HlsAccessLeaseId(lease.to_string());
let access = test_hls_access_context(fixture.proxy_session_id.clone(), access_lease_id.clone());
prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &access_lease_id).await;
let response = super::super::try_hls_cache_canonical_manifest_response(
&fixture.app_state,
&test_fingerprint(),
&access,
&fixture.proxy_session_id,
&access_lease_id,
HlsAccessLeaseState::Pending,
super::super::HlsCacheManifestOrigin {
raw_request_url: &fixture.request_url,
session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&fixture.request_url),
input: &fixture.input,
origin_source: super::super::build_hls_origin_source(&fixture.input, "12345"),
},
HeaderMap::new(),
None,
"/m3u-stream/live/hls-user/hls-pass/12345.m3u8",
super::super::HlsManifestRefreshOrdering::Background,
)
.await
.expect("shared HLS response");
assert_eq!(response.status(), StatusCode::OK);
assert!(!response.headers().contains_key(header::RETRY_AFTER));
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
assert!(!body.contains("#EXT-X-ENDLIST"), "short origin must remain live: {body}");
assert!(body.contains("/hls/shared/live/"));
body
}
async fn assert_short_media(fixture: &ShortWindowFixture, body: &str) {
for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response_body(response).await.as_ref(), b"short-window-media");
}
}
#[tokio::test]
async fn one_segment_origin_publishes_live_manifest_and_segment() {
let fixture = short_window_fixture(1).await;
let body = open_short_lease(&fixture, "short-one").await;
assert_eq!(media_uri_count(&body), 1);
assert_short_media(&fixture, &body).await;
assert_eq!(fixture.origin.manifest_request_count(), 3);
}
#[tokio::test]
async fn two_segment_origin_publishes_live_manifest_and_segments() {
let fixture = short_window_fixture(2).await;
let body = open_short_lease(&fixture, "short-two").await;
assert_eq!(media_uri_count(&body), 2);
assert_short_media(&fixture, &body).await;
assert_eq!(fixture.origin.manifest_request_count(), 3);
}
#[tokio::test]
async fn twenty_actual_leases_share_the_same_short_origin_qualification() {
let mut counts = Vec::new();
for leases in [1, 20] {
let fixture = Arc::new(short_window_fixture(1).await);
let mut tasks = tokio::task::JoinSet::new();
for lease in 0..leases {
let fixture = Arc::clone(&fixture);
tasks.spawn(async move { open_short_lease(&fixture, &format!("short-shared-{lease}")).await });
}
while let Some(result) = tasks.join_next().await {
let body = result.unwrap();
assert_eq!(media_uri_count(&body), 1);
assert_short_media(&fixture, &body).await;
}
counts.push(fixture.origin.manifest_request_count());
let before_new_lease = fixture.origin.manifest_request_count();
open_short_lease(&fixture, "later-short-lease").await;
assert_eq!(fixture.origin.manifest_request_count(), before_new_lease);
}
assert_eq!(counts, vec![3, 3]);
}
#[tokio::test]
async fn ten_short_refresh_cycles_keep_serving_live_media_and_immutable_old_routes() {
assert_ten_short_refresh_cycles([3, 1, 3, 1, 3, 1, 3, 1, 3, 1]).await;
}
#[tokio::test]
async fn continuously_advancing_one_segment_origin_serves_ten_refresh_cycles() {
assert_ten_short_refresh_cycles([1; 10]).await;
}
async fn assert_ten_short_refresh_cycles(counts: [usize; 10]) {
let fixture = short_window_fixture(1).await;
let initial = open_short_lease(&fixture, "short-reloads").await;
assert_short_media(&fixture, &initial).await;
let uri = format!("/hls/shared/live/{}/short-reloads/manifest.m3u8", fixture.proxy_session_id.0);
let session =
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
let mut previous = initial.clone();
let mut previous_last = session.read().await.last_rendered_manifest.as_ref().unwrap().last_proxy_seq;
for cycle in 1..=10 {
fixture.head.store(100 + cycle * 4, Ordering::SeqCst);
fixture.count.store(counts[usize::try_from(cycle - 1).unwrap()], Ordering::SeqCst);
{
let mut session = session.write().await;
session.origin_refresh.next_fetch_allowed_at_ms = 0;
}
let mut current = None;
for _ in 0..100 {
let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
assert!(!body.contains("#EXT-X-ENDLIST"));
if body != previous {
current = Some(body);
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let body = current.expect("normal refresh advances the short window");
assert!(manifest_media_sequence(&body) >= manifest_media_sequence(&previous));
previous = body.clone();
assert_short_media(&fixture, &body).await;
let lease = fixture
.app_state
.hls
.proxy
.access_lease_response_snapshot(
&HlsAccessLeaseId("short-reloads".to_string()),
&fixture.proxy_session_id,
super::super::current_time_millis(),
)
.await
.unwrap();
let snapshot = lease.last_manifest_snapshot.unwrap();
assert!(snapshot.last_proxy_seq > previous_last);
previous_last = snapshot.last_proxy_seq;
assert_eq!(snapshot.playlist_duration_ms, u64::try_from(media_uri_count(&body)).unwrap() * 2_500);
}
assert_eq!(fixture.origin.manifest_request_count(), 13);
assert_short_media(&fixture, &initial).await;
let invalid = uri.replace("short-reloads", "missing-lease");
assert_ne!(get_response(Arc::clone(&fixture.app_state), &invalid, None).await.status(), StatusCode::OK);
}
#[tokio::test]
async fn short_manifests_keep_map_key_and_segment_routes_on_later_publications() {
let fixture = short_window_resource_fixture(2, ShortWindowResources::EncryptedWithMap).await;
let initial = open_short_lease(&fixture, "short-resources").await;
let session =
fixture.app_state.hls.proxy.sessions().get_by_proxy_session_id(&fixture.proxy_session_id).await.unwrap();
for uri in initial.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
response_body(response).await;
}
let mut bodies = vec![initial];
fixture.head.store(102, Ordering::SeqCst);
session.write().await.origin_refresh.next_fetch_allowed_at_ms = 0;
let uri = format!("/hls/shared/live/{}/short-resources/manifest.m3u8", fixture.proxy_session_id.0);
for _ in 0..100 {
let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
if body != bodies[0] {
bodies.push(body);
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_eq!(bodies.len(), 2, "a second short publication must advance");
for body in bodies {
assert!(!body.contains("#EXT-X-ENDLIST"));
for (tag, expected) in [("#EXT-X-MAP:", &b"short-window-map"[..]), ("#EXT-X-KEY:", AES_TEST_KEY_BYTES)] {
let uri = body
.lines()
.find(|line| line.starts_with(tag))
.unwrap()
.split_once("URI=\"")
.unwrap()
.1
.split_once('"')
.unwrap()
.0;
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response_body(response).await.as_ref(), expected);
}
for uri in body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
let response = get_response(Arc::clone(&fixture.app_state), uri, None).await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response_body(response).await.as_ref(),
encrypt_test_aes128_cbc_pkcs7(b"short-window-media", AES_TEST_KEY_BYTES, [1; 16])
);
}
}
assert_eq!(fixture.origin.manifest_request_count(), 4);
}
@@ -420,7 +420,7 @@ async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest()
assert_eq!(binding.input_name.as_ref(), "test-input");
assert_eq!(binding.account_name.as_ref(), "test-input");
}
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
File diff suppressed because it is too large Load Diff
@@ -491,7 +491,7 @@ async fn tokens_without_entry_kind_keep_bad_request_without_session() {
let response = fixture.token_request(&client("10.0.0.1", 50_121), &token, HeaderMap::new()).await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
assert!(fixture.origin.requests().is_empty());
assert_eq!(fixture.origin.requests().len(), 0);
}
#[tokio::test]
@@ -507,7 +507,7 @@ async fn shared_hls_cache_on_never_recreates_from_token() {
);
let response = fixture.token_request(&client("10.0.0.1", 50_131), &token, HeaderMap::new()).await;
assert_ne!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(fixture.origin.requests().is_empty());
assert_eq!(fixture.origin.requests().len(), 0);
assert!(fixture.app_state.active_users.get_and_update_user_session("hls-user", hint).await.is_none());
}
@@ -21,6 +21,9 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_hls_origin_io_context(
reservation_ttl_secs: 60,
preacquired_provider_handle: None,
started_generation: None,
prepared_account: None,
playback_kind: tuliprox_core::model::PlaybackKind::LiveHls,
account_runtime: None,
}
}
@@ -156,8 +156,8 @@ fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() {
#[tokio::test]
async fn hls_account_overlap_reclaim_preempts_speculative_session() {
let app_state = test_app_state();
let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() };
let input = overlap_provider_input();
let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await;
let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await;
let winner_proxy_session_id = winner.read().await.proxy_session_id.clone();
@@ -175,7 +175,9 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() {
}
let loser_generation = loser.read().await.activity.origin_work_generation;
super::super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await;
winner.write().await.mark_authorized_media_access(10_000);
winner.write().await.activity.last_delivered_media_at_ms = Some(10_000);
drop(prepare_overlap_origin(&app_state, &input, &winner, 10_000).await);
assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await);
assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await);
@@ -191,8 +193,8 @@ async fn hls_account_overlap_reclaim_preempts_speculative_session() {
#[tokio::test]
async fn hls_account_overlap_promotes_speculative_session_after_soft_window() {
let app_state = test_app_state();
let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() };
let input = overlap_provider_input();
let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await;
let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await;
let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone();
@@ -210,7 +212,9 @@ async fn hls_account_overlap_promotes_speculative_session_after_soft_window() {
}
let displaced_generation = displaced.read().await.activity.origin_work_generation;
super::super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await;
promoted.write().await.mark_authorized_media_access(20_001);
promoted.write().await.activity.last_delivered_media_at_ms = Some(20_001);
drop(prepare_overlap_origin(&app_state, &input, &promoted, 20_001).await);
let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone();
assert!(matches!(
@@ -262,8 +266,7 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work(
.await
.expect("interactive work should use soft-active overlap before grace");
let binding =
prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared");
let binding = new_session.read().await.origin_account_binding.clone().expect("speculative binding is published");
assert_eq!(binding.account_name, input.name);
assert!(matches!(
&binding.binding_mode,
@@ -273,11 +276,15 @@ async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work(
} if displaced_proxy_session_id == &old_proxy_session_id
));
assert!(matches!(
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
prepared_origin
.preacquired_origin_account_handle
.as_ref()
.and_then(tuliprox_session::ManagedProviderHandle::handle)
.map(|handle| &handle.allocation),
Some(super::super::ProviderAllocation::Available(_))
));
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
}
#[tokio::test]
@@ -324,14 +331,16 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() {
.await
.expect("normal HLS policy should preempt active soft HLS binding");
let new_binding = prepared_origin
.origin_account_binding_to_store
.as_ref()
.expect("preempting session should receive active binding");
let new_binding =
normal_session.read().await.origin_account_binding.clone().expect("preempting session receives active binding");
assert_eq!(new_binding.account_name, input.name);
assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active));
assert!(matches!(
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
prepared_origin
.preacquired_origin_account_handle
.as_ref()
.and_then(tuliprox_session::ManagedProviderHandle::handle)
.map(|handle| &handle.allocation),
Some(super::super::ProviderAllocation::Available(_))
));
let soft_session = soft_session.read().await;
@@ -345,7 +354,7 @@ async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() {
));
drop(soft_session);
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
}
#[tokio::test]
@@ -437,7 +446,6 @@ async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal
let result = super::super::prepare_hls_origin_policy_preempt_runtime(
&app_state,
&soft_session,
&input,
"http://account.example.com/live/account-user/account-pass/soft.m3u8",
"http://account.example.com/live/account-user/account-pass/soft.m3u8",
@@ -496,12 +504,18 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai
.expect("interactive work can use grace when normal acquire and overlap fail");
assert!(matches!(
prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation),
prepared_origin
.preacquired_origin_account_handle
.as_ref()
.and_then(tuliprox_session::ManagedProviderHandle::handle)
.map(|handle| &handle.allocation),
Some(super::super::ProviderAllocation::GracePeriod(_))
));
assert_eq!(
prepared_origin
.origin_account_binding_to_store
session
.read()
.await
.origin_account_binding
.as_ref()
.expect("grace binding should still bind the selected account")
.account_name
@@ -509,7 +523,7 @@ async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fai
input.name.as_ref()
);
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
app_state.connection_manager.release_provider_handle(Some(occupied));
}
@@ -715,3 +729,40 @@ fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_
super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted
);
}
async fn prepare_overlap_origin(
app: &Arc<crate::api::model::AppState>,
input: &ConfigInput,
session: &tuliprox_hls::api::HlsSessionHandle,
now_ms: u64,
) -> super::super::session::PreparedHlsOriginRuntime {
let id = session.read().await.proxy_session_id.clone();
super::super::prepare_hls_origin_runtime(
app,
session,
input,
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
&id,
&test_fingerprint(),
ConnectionKind::Normal,
0,
super::super::HlsOriginWorkKind::Manifest,
super::super::HlsOriginWorkClass::ManifestInteractive,
now_ms,
)
.await
.unwrap()
}
#[tokio::test]
async fn healthy_prepare_detaches_an_unprotected_expired_binding_before_reacquisition() {
let input = overlap_provider_input();
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
let session = create_bound_hls_test_session(&app, &input, "expired-protection", "account-a", 1_000).await;
let generation = session.read().await.activity.origin_work_generation;
drop(prepare_overlap_origin(&app, &input, &session, 10_000).await);
let current = session.read().await;
assert!(current.activity.origin_work_generation > generation);
assert!(current.origin_account_binding.as_ref().unwrap().generation > 0);
}
@@ -21,6 +21,33 @@ use http_body_util::BodyExt;
use shared::model::{HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster};
use std::{sync::Arc, time::Duration};
#[tokio::test]
async fn origin_resource_no_account_available_includes_retry_after() {
let app_state = test_app_state();
let access_context = test_hls_access_context(
ProxySessionId("shared-hls-session".to_string()),
HlsAccessLeaseId("access-lease".to_string()),
);
for reason in [
super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted,
super::super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted,
] {
let response = super::super::hls_origin_runtime_resource_failure_response(
&app_state,
&access_context,
super::super::HlsOriginRuntimeAcquireError::NoAccountAvailable { reason },
);
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{reason:?}");
assert_eq!(
response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()),
Some("2"),
"{reason:?}"
);
}
}
#[tokio::test]
async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() {
let input = single_hls_provider_input("known-account");
@@ -52,7 +79,7 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho
}
let old_generation = session.read().await.activity.origin_work_generation;
super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await;
super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 6_001).await;
{
let session = session.read().await;
@@ -77,17 +104,16 @@ async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_autho
0,
super::super::HlsOriginWorkKind::Manifest,
super::super::HlsOriginWorkClass::ManifestInteractive,
4_100,
6_100,
)
.await
.expect("authorized origin work can reacquire a detached binding");
let binding =
prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller");
let binding = session.read().await.origin_account_binding.clone().expect("new binding is published atomically");
assert_eq!(binding.account_name.as_ref(), "account-a");
assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active));
assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8");
app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle);
app_state.connection_manager.release_managed_provider_handle(prepared_origin.preacquired_origin_account_handle);
}
#[tokio::test]
@@ -122,6 +148,70 @@ async fn hls_account_binding_without_media_activity_expires_after_startup_window
assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000);
}
#[tokio::test]
async fn fresh_hls_binding_has_bounded_startup_protection_despite_old_media_delivery() {
let input = overlap_provider_input();
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await;
{
let mut s = session.write().await;
s.target_duration = Some(11);
s.activity.last_authorized_media_at_ms = Some(70_000);
s.activity.last_delivered_media_at_ms = Some(70_000);
}
let proxy_id = session.read().await.proxy_session_id.clone();
for now in [200_001, 204_999, 205_000] {
let prepared = super::super::prepare_hls_origin_runtime(
&app,
&session,
&input,
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
"http://root.example.com/live/root-user/root-pass/12345.m3u8",
&proxy_id,
&test_fingerprint(),
ConnectionKind::Normal,
0,
super::super::HlsOriginWorkKind::Resource,
super::super::HlsOriginWorkClass::Demand,
now,
)
.await
.unwrap();
let s = session.read().await;
let binding = s.origin_account_binding.as_ref().unwrap();
assert!(binding.is_active());
assert_eq!(binding.pinned_at_ms, 200_000, "preparing the same binding cannot extend startup protection");
assert_eq!(binding.generation, 0);
assert_eq!(s.activity.last_delivered_media_at_ms, Some(70_000));
drop(s);
app.connection_manager.release_managed_provider_handle(prepared.preacquired_origin_account_handle);
}
super::super::detach_unprotected_hls_origin_account_bindings(&app, 205_001).await;
assert!(
session.read().await.origin_account_binding.as_ref().unwrap().is_detached(),
"no media progress: releasable after five seconds"
);
}
#[tokio::test]
async fn hls_binding_expired_startup_protects_running_origin_work() {
let input = overlap_provider_input();
let app = test_app_state_with_inputs(vec![Arc::new(input.clone())]);
let session = create_bound_hls_test_session(&app, &input, "12345", "account-a", 200_000).await;
let generation = {
let mut s = session.write().await;
s.target_duration = Some(11);
s.activity.last_authorized_media_at_ms = Some(70_000);
s.activity.last_delivered_media_at_ms = Some(70_000);
s.start_origin_work()
};
super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_000).await;
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_active());
assert!(session.write().await.finish_origin_work(generation));
super::super::detach_unprotected_hls_origin_account_bindings(&app, 210_001).await;
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached());
}
#[tokio::test]
async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() {
let app_state = test_app_state();
@@ -145,6 +235,7 @@ async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_d
host_local_index: 123,
},
proxy_seq: 123,
media_start_ms: 0,
duration_ms: 4_000,
proxy_file_ext: "ts".to_string(),
content_type: "video/mp2t".to_string(),
@@ -95,7 +95,7 @@ async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues
assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. }));
}
assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner")));
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
#[tokio::test]
@@ -0,0 +1,287 @@
use super::{
super::origin_refresh_request::{hls_origin_refresh_request, HlsOriginRefreshOptions},
manifest_entry_freshness::open_manifest_for_new_lease,
origin_account_rebind::{
enable_isolated_rebind_cache, reduced_retry_origin, resume_origin_media, ReducedRetryOriginControl,
},
*,
};
use shared::model::{HlsManifestRecoveryBurstLevel, HlsStartupConfigDto, HlsStartupMode};
use tuliprox_core::utils::current_time_millis;
use tuliprox_hls::api::{
trigger_origin_refresh_sync, HlsManifestCommitRequirement, HlsOriginAccountSnapshot, HlsPreparedOriginAccount,
};
const WAIT: Duration = Duration::from_secs(15);
async fn ready(app: &Arc<AppState>, session: &HlsSessionHandle, sequences: &[u64]) {
tokio::time::timeout(WAIT, async {
loop {
let s = session.read().await;
if sequences.iter().all(|seq| {
s.segments.get(seq).is_some_and(|entry| matches!(entry.status, SegmentCacheStatus::Ready { .. }))
}) && s.active_segment_fetches == 0
&& !s.origin_refresh.in_flight
&& s.activity.active_origin_work_count.load(Ordering::Acquire) == 0
{
break;
}
drop(s);
tokio::task::yield_now().await;
}
})
.await
.expect("autonomous segment work and singleflight complete");
assert_eq!(app.active_provider.get_provider_connections_count(), 0);
}
async fn deliver(app: &Arc<AppState>, session: &HlsSessionHandle, manifest: &str, expected: &[u64]) {
assert!(!manifest.contains("/cvs/hls/"));
assert!(!manifest.contains("#EXT-X-ENDLIST"));
for sequence in expected {
let (proxy_seq, url) = {
let s = session.read().await;
let (seq, entry) =
s.segments.iter().find(|(_, entry)| entry.origin_key.host_local_sequence == *sequence).unwrap();
(*seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone())
};
let uri = manifest
.lines()
.find(|line| line.ends_with(&format!("/{proxy_seq:06}.ts")))
.expect("segment URI is published by Tuliprox");
let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await;
assert_eq!(response.status(), StatusCode::OK);
let bytes = response_body(response).await;
let path = reqwest::Url::parse(&url).unwrap().path().to_string();
assert_eq!(bytes.as_ref(), resume_origin_media(&path));
eprintln!("app delivered resource={path} bytes={}", bytes.len());
}
}
#[expect(clippy::too_many_lines, reason = "Keep the ordered end-to-end recovery scenario and its assertions together.")]
async fn repaired_reduced_retry(mode: HlsStartupMode) {
let control = Arc::new(ReducedRetryOriginControl::default());
let origin = reduced_retry_origin(Arc::clone(&control)).await;
let cache = tempfile::tempdir().unwrap();
let mut input = overlap_provider_input();
input.name = Arc::from("account-a");
input.priority = 0;
input.url.clone_from(&origin.base_url);
let b = &mut input.aliases.as_mut().unwrap()[0];
b.name = Arc::from("account-b");
b.priority = 10;
b.url.clone_from(&origin.base_url);
let app = test_app_state_with_hls_proxy_and_inputs(
Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)),
vec![Arc::new(input.clone())],
);
let available_b = HlsOriginAccountBinding::new(
Arc::clone(&input.name),
Arc::from("account-b"),
&ProxySessionId("availability-check".into()),
current_time_millis(),
);
assert_eq!(
tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b),
tuliprox_hls::api::HlsOriginAccountStatus::Known
);
enable_isolated_rebind_cache(&app, cache.path());
let mut config = app.app_config.config.load_full().as_ref().clone();
let hls = config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap();
hls.startup = (&HlsStartupConfigDto { mode, ..Default::default() }).into();
hls.manifest_recovery_burst.level = HlsManifestRecoveryBurstLevel::Beast;
app.app_config.config.store(Arc::new(config));
enable_channel_unavailable_custom_response(&app);
app.hls.proxy.update_config(&app.app_config).await;
create_active_hls_user_session(&app).await;
let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url);
let lease = "reduced-retry-baseline";
let _initial = open_manifest_for_new_lease(&app, &input, &url, lease).await;
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
ready(&app, &session, &[0, 1, 2, 3, 4, 5]).await;
let response = try_test_hls_cached_manifest_response(
&app,
&session,
&HlsAccessLeaseId(lease.to_string()),
HlsAccessLeaseState::Pending,
&StripConfig { mode: HlsStripMode::Segments, value: 0 },
None,
HlsCachedManifestOptions::initial(Duration::ZERO),
)
.await
.expect("complete baseline is published to the playback lease");
assert_eq!(response.status(), StatusCode::OK);
let first = String::from_utf8(response_body(response).await.to_vec()).unwrap();
deliver(&app, &session, &first, &[1000]).await;
tokio::time::timeout(WAIT, async {
let proxy_id = session.read().await.proxy_session_id.clone();
loop {
let snapshot = app
.hls
.proxy
.access_lease_response_snapshot(&HlsAccessLeaseId(lease.to_string()), &proxy_id, current_time_millis())
.await
.unwrap();
if snapshot.playback_cursor.highest_contiguous_completed_proxy_seq == Some(0) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("baseline body completion is recorded before recovery pressure is evaluated");
let (binding, baseline, commits, proxy_id) = {
let s = session.read().await;
assert!(s.activity.last_authorized_media_at_ms.is_some());
assert!(s.activity.last_delivered_media_at_ms.is_some());
assert!(s.established_manifest_recovery_binding().is_some());
(
s.origin_account_binding.clone().unwrap(),
s.origin_to_proxy.clone(),
s.origin_manifest_commits(),
s.proxy_session_id.clone(),
)
};
assert_eq!(binding.account_name.as_ref(), "account-a");
eprintln!(
"{mode:?} baseline account={} binding_generation={} pinned_at_ms={} proxy_session={}",
binding.account_name, binding.generation, binding.pinned_at_ms, proxy_id.0
);
session.write().await.origin_control.path_condition = tuliprox_hls::HlsOriginPathCondition::HardFetchFailure;
let directive =
match tuliprox_hls::api::hls_manifest_acceptance_directive_for_session(&app.hls_ctx(), &session, &proxy_id)
.await
{
tuliprox_hls::api::HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive,
other => panic!("recovery pressure must evaluate: {other:?}"),
};
eprintln!("app baseline directive={directive:?}");
assert!(directive.recovery_pressure_guard.is_some(), "directive has current lease/session evidence");
assert_eq!(directive.trigger, tuliprox_hls::HlsManifestAcceptanceTrigger::RecoveryRequired);
let mut io = super::provider_admission::test_hls_origin_io_context(&app);
io.account_runtime = session.read().await.origin_account_runtime.clone();
let snapshot = HlsOriginAccountSnapshot::from_session(&*session.read().await);
let mut request = hls_origin_refresh_request(
&app,
&session,
HlsPreparedOriginAccount { entry: LiveHlsOriginEntry::parse(&url).unwrap(), snapshot },
HlsOriginRefreshOptions {
headers: HeaderMap::new(),
origin_io: io,
manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed,
acceptance_directive: directive,
access_lease_id: Some(HlsAccessLeaseId(lease.to_string())),
rewrite_secret: app.get_encrypt_secret().to_vec(),
now_ms: current_time_millis(),
},
);
request.now_ms = session.read().await.origin_refresh.next_fetch_allowed_at_ms.max(current_time_millis());
control.recovering.store(true, Ordering::Release);
let refresh = tokio::spawn(trigger_origin_refresh_sync(request));
tokio::time::timeout(WAIT, control.retry_started.notified()).await.expect("reduced HTTP attempt 13 starts");
let episode_before = {
let s = session.read().await;
assert_eq!(s.origin_to_proxy, baseline);
assert_eq!(s.origin_manifest_commits(), commits, "first actual conflict commit did not mutate the timeline");
let episode = s.origin_control.acceptance_episode.clone().unwrap();
assert_eq!(episode.full_bursts_completed, 1);
assert!(episode.exhaustion_reason().is_some());
assert!(episode.deterministic_conflict_receipt().is_none());
eprintln!(
"{mode:?} before response 13: generation={} full_bursts={} reason={:?} timing={:?}",
episode.generation.0,
episode.full_bursts_completed,
episode.exhaustion_reason(),
episode.timing()
);
episode
};
let prepare_gate = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test();
control.release_retry.notify_one();
tokio::time::timeout(WAIT, prepare_gate.entered.notified())
.await
.expect("repaired reduced retry commits and schedules media");
let scheduled = {
let s = session.read().await;
let mut resources = s
.segment_prefetch_queue
.proxy_seqs()
.iter()
.map(|seq| {
let entry = &s.segments[seq];
(
*seq,
entry.origin_key.host_local_sequence,
entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone(),
)
})
.collect::<Vec<_>>();
resources.sort();
assert_eq!(resources.iter().map(|(_, seq, _)| *seq).collect::<Vec<_>>(), vec![1006, 1007, 1008]);
assert_eq!(s.origin_manifest_commits(), commits + 1);
assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation);
assert_eq!(s.origin_control.acceptance_generation, episode_before.generation);
assert!(
s.origin_control.acceptance_episode.is_none(),
"successful refresh bookkeeping consumes the completed episode"
);
eprintln!("{mode:?} committed generation={} scheduled={resources:?}", episode_before.generation.0);
resources
};
prepare_gate.prepare.notify_one();
tokio::time::timeout(WAIT, prepare_gate.prepared.notified()).await.unwrap();
prepare_gate.proceed.notify_one();
let seqs = scheduled.iter().map(|(seq, _, _)| *seq).collect::<Vec<_>>();
ready(&app, &session, &seqs).await;
tokio::time::timeout(WAIT, refresh).await.unwrap().unwrap();
assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13);
{
let requests = origin.requests.lock().await;
for (_, _, url) in &scheduled {
let path = reqwest::Url::parse(url).unwrap().path().to_string();
let count = requests.iter().filter(|request| request.starts_with(&format!("GET {path} "))).count();
assert!(count > 0, "queued resource was fetched before any client demand");
eprintln!("{mode:?} autonomous origin GET {path} requests={count} status=Ready");
}
assert!(requests.iter().all(|request| !request.contains("account-user")), "B remains unused");
}
let response = try_test_hls_cached_manifest_response(
&app,
&session,
&HlsAccessLeaseId(lease.to_string()),
HlsAccessLeaseState::Activated,
&StripConfig { mode: HlsStripMode::Segments, value: 0 },
None,
HlsCachedManifestOptions::initial(Duration::ZERO),
)
.await
.expect("repaired live manifest is published");
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).unwrap();
deliver(&app, &session, &body, &[1006, 1007, 1008]).await;
let s = session.read().await;
assert_eq!(s.origin_account_binding.as_ref().unwrap().generation, binding.generation);
assert_eq!(s.origin_account_binding.as_ref().unwrap().account_name, binding.account_name);
assert_eq!(s.origin_account_binding.as_ref().unwrap().pinned_at_ms, binding.pinned_at_ms);
assert_eq!(s.origin_account_binding.as_ref().unwrap().provider_binding_tag, binding.provider_binding_tag);
assert_eq!(control.recovery_requests.load(Ordering::Acquire), 13);
eprintln!(
"{mode:?} complete: recovery_requests=13 account={} binding_generation={} origin_work=0 singleflight=false",
binding.account_name, binding.generation
);
assert_eq!(s.proxy_session_id, proxy_id);
assert!(!s.origin_refresh.in_flight);
assert_eq!(s.active_segment_fetches, 0);
assert_eq!(s.activity.active_origin_work_count.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn hls_reduced_retry_delivers_repaired_origin_media_optimistic() {
repaired_reduced_retry(HlsStartupMode::Optimistic).await;
}
#[tokio::test]
async fn hls_reduced_retry_delivers_repaired_origin_media_conservative() {
repaired_reduced_retry(HlsStartupMode::Conservative).await;
}
@@ -0,0 +1,294 @@
use super::{
manifest_entry_freshness::open_manifest_for_new_lease,
origin_account_rebind::{enable_isolated_rebind_cache, resume_origin, resume_origin_media},
*,
};
use shared::model::{HlsStartupConfigDto, HlsStartupMode};
use std::sync::atomic::AtomicU64;
use tuliprox_core::utils::current_time_millis;
const PAUSE_MS: u64 = 130_000;
const WAIT: Duration = Duration::from_secs(5);
async fn settle_origin(session: &HlsSessionHandle) {
tokio::time::timeout(WAIT, async {
loop {
let s = session.read().await;
if !s.origin_refresh.in_flight
&& s.active_segment_fetches == 0
&& s.activity.active_origin_work_count.load(Ordering::Acquire) == 0
{
break;
}
drop(s);
tokio::task::yield_now().await;
}
})
.await
.expect("origin work settles before playback is stopped");
}
/// Model wall-clock absence without erasing actual media delivery, or relying on
/// Tokio's clock (account preparation and HTTP serving also read `SystemTime`).
async fn pause_playback(app: &Arc<AppState>, session: &HlsSessionHandle, lease: &str) {
settle_origin(session).await;
assert!(app
.hls
.proxy
.segment_worker_pool()
.access_leases()
.write()
.await
.remove_access_lease(&HlsAccessLeaseId(lease.to_string()))
.is_some());
app.active_users.release_session_streams_and_counted_reservation("hls-user", "hls-session-token").await;
let now = current_time_millis();
let proxy_id = {
let mut s = session.write().await;
let age = |time: &mut Option<u64>| *time = time.map(|time| time.saturating_sub(PAUSE_MS));
age(&mut s.activity.last_authorized_manifest_at_ms);
age(&mut s.activity.last_authorized_media_at_ms);
age(&mut s.activity.last_delivered_media_at_ms);
s.last_client_access_at_ms = s.last_client_access_at_ms.saturating_sub(PAUSE_MS);
let binding = s.origin_account_binding.as_mut().unwrap();
binding.pinned_at_ms = binding.pinned_at_ms.saturating_sub(PAUSE_MS);
age(&mut binding.last_origin_io_at_ms);
age(&mut binding.last_reservation_refresh_at_ms);
age(&mut s.origin_refresh.last_fetch_started_at_ms);
age(&mut s.origin_refresh.last_fetch_finished_at_ms);
age(&mut s.origin_refresh.last_success_at_ms);
age(&mut s.origin_refresh.last_error_at_ms);
s.origin_refresh.next_fetch_allowed_at_ms = s.origin_refresh.next_fetch_allowed_at_ms.saturating_sub(PAUSE_MS);
if let Some(rendered) = s.last_rendered_manifest.as_mut() {
rendered.rendered_at_ms = rendered.rendered_at_ms.saturating_sub(PAUSE_MS);
rendered.valid_until_ms = rendered.valid_until_ms.saturating_sub(PAUSE_MS);
}
assert!(s.activity.last_delivered_media_at_ms.is_some_and(|last| now - last >= PAUSE_MS));
assert!(!s.can_expire_idle_session(now, 300_000), "retained session is inside its 300-second timeout");
s.proxy_session_id.clone()
};
app.hls
.proxy
.sync_session_access_lease_count_and_detach_if_needed(
&app.active_users,
&app.active_provider,
session,
&proxy_id,
now,
)
.await;
super::super::detach_unprotected_hls_origin_account_bindings(app, now).await;
assert!(session.read().await.origin_account_binding.as_ref().unwrap().is_detached());
assert_eq!(app.active_provider.get_provider_connections_count(), 0, "pause releases provider slots");
for account in ["account-a", "account-b"] {
assert_eq!(
app.active_provider.provider_lease_usage(&Arc::from(account)).total(),
0,
"no logical slot remains for {account}"
);
}
}
fn segment_uris(manifest: &str) -> Vec<&str> {
assert!(manifest.starts_with("#EXTM3U"), "live playlist: {manifest}");
assert!(!manifest.contains("/cvs/hls/"), "replacement video must never satisfy resume: {manifest}");
assert!(!manifest.contains("#EXT-X-ENDLIST"), "resume remains live");
let uris = manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).collect::<Vec<_>>();
assert!(!uris.is_empty(), "live media playlist must contain media");
uris
}
async fn deliver_origin_segment(app: &Arc<AppState>, session: &HlsSessionHandle, manifest: &str, head: u64) {
let uri = segment_uris(manifest)[0];
let seq = uri.rsplit('/').next().unwrap().trim_end_matches(".ts").parse::<u64>().unwrap();
let origin_url = {
let s = session.read().await;
let segment = &s.segments[&seq];
assert!((head..head + 6).contains(&segment.origin_key.host_local_sequence), "current origin window");
segment.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone()
};
let path = reqwest::Url::parse(&origin_url).unwrap().path().to_string();
let response = get_response(Arc::clone(app), uri.strip_prefix("/iptv").unwrap_or(uri), None).await;
assert_eq!(response.status(), StatusCode::OK);
let body = response_body(response).await;
assert_eq!(body.as_ref(), resume_origin_media(&path), "client receives the exact current origin object");
let s = session.read().await;
assert!(s.activity.last_authorized_media_at_ms.is_some());
assert!(s.activity.last_delivered_media_at_ms.is_some(), "fully consume the Tuliprox response before pausing");
eprintln!(
"delivered origin={path} bytes={} authorized={:?} delivered={:?}",
body.len(),
s.activity.last_authorized_media_at_ms,
s.activity.last_delivered_media_at_ms
);
}
#[expect(clippy::too_many_lines, reason = "Keep the ordered pause/resume scenario and its assertions together.")]
async fn same_account_resume(mode: HlsStartupMode, resumes: usize) {
let head = Arc::new(AtomicU64::new(1113));
let origin = resume_origin(Arc::clone(&head)).await;
let cache = tempfile::tempdir().unwrap();
let mut input = overlap_provider_input();
input.name = Arc::from("account-a");
input.priority = 0;
input.url.clone_from(&origin.base_url);
let b = &mut input.aliases.as_mut().unwrap()[0];
b.name = Arc::from("account-b");
b.priority = 10;
b.url.clone_from(&origin.base_url);
let app = test_app_state_with_hls_proxy_and_inputs(
Arc::new(HlsProxyManager::with_cache_settings(cache.path(), 300)),
vec![Arc::new(input.clone())],
);
let available_b = HlsOriginAccountBinding::new(
Arc::clone(&input.name),
Arc::from("account-b"),
&ProxySessionId("availability-check".into()),
current_time_millis(),
);
assert_eq!(
tuliprox_hls::api::hls_origin_account_status(&app.hls_ctx(), &available_b),
tuliprox_hls::api::HlsOriginAccountStatus::Known
);
enable_isolated_rebind_cache(&app, cache.path());
let mut config = app.app_config.config.load_full().as_ref().clone();
config.reverse_proxy.as_mut().unwrap().hls_cache.as_mut().unwrap().startup =
(&HlsStartupConfigDto { mode, ..HlsStartupConfigDto::default() }).into();
app.app_config.config.store(Arc::new(config));
enable_channel_unavailable_custom_response(&app);
app.hls.proxy.update_config(&app.app_config).await;
assert_eq!(app.hls.proxy.session_idle_timeout_ms(), 300_000);
create_active_hls_user_session(&app).await;
let url = format!("{}/live/root-user/root-pass/12345.m3u8", origin.base_url);
let key = super::super::build_hls_origin_source(&input, "12345").session_key();
let mut lease = "resume-initial".to_string();
let first = open_manifest_for_new_lease(&app, &input, &url, &lease).await;
let session = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
deliver_origin_segment(&app, &session, &first, 1113).await;
settle_origin(&session).await;
let proxy_id = session.read().await.proxy_session_id.clone();
for cycle in 0..resumes {
let old = session.read().await.origin_account_binding.clone().unwrap();
assert_eq!(old.account_name.as_ref(), "account-a");
pause_playback(&app, &session, &lease).await;
let current_head = 1126 + u64::try_from(cycle).unwrap() * 13;
head.store(current_head, Ordering::Release);
lease = format!("resume-{cycle}");
create_active_hls_user_session(&app).await;
let boundary = origin.requests.lock().await.len();
let started_at = current_time_millis();
let barrier = app.hls.proxy.segment_worker_pool().pause_next_origin_prepare_for_test();
let (task_app, task_input, task_url, task_lease) =
(Arc::clone(&app), input.clone(), url.clone(), lease.clone());
let opening =
tokio::spawn(
async move { open_manifest_for_new_lease(&task_app, &task_input, &task_url, &task_lease).await },
);
tokio::time::timeout(WAIT, barrier.entered.notified()).await.expect("manifest commit reaches scheduler");
let retained = app.hls.proxy.sessions().get_by_key(&key).await.unwrap();
assert!(Arc::ptr_eq(&session, &retained), "same shared session allocation");
let (rebound, scheduled, delivered_before_prepare) = {
let s = session.read().await;
assert_eq!(s.key, key);
assert_eq!(s.proxy_session_id, proxy_id);
assert_eq!(s.target_duration, Some(11));
let rebound = s.origin_account_binding.clone().unwrap();
assert_eq!(rebound.account_name, old.account_name, "B is available; A is actually rebound");
assert!(rebound.is_active());
assert!(rebound.generation > old.generation);
assert!(rebound.pinned_at_ms >= started_at);
let mut scheduled = s
.segment_prefetch_queue
.proxy_seqs()
.into_iter()
.map(|seq| {
let entry = &s.segments[&seq];
assert!((current_head..current_head + 6).contains(&entry.origin_key.host_local_sequence));
(seq, entry.origin_fetch_ref.as_ref().unwrap().resolved_origin_url.clone())
})
.collect::<Vec<_>>();
scheduled.sort();
assert!(!scheduled.is_empty(), "capture jobs immediately after commit, before preparation");
eprintln!("{mode:?} cycle={cycle} shared={key:?} proxy={proxy_id:?} account={} binding={}->{} pinned={} scheduled={scheduled:?}", rebound.account_name, old.generation, rebound.generation, rebound.pinned_at_ms);
(rebound, scheduled, s.activity.last_delivered_media_at_ms)
};
barrier.prepare.notify_one();
tokio::time::timeout(WAIT, barrier.prepared.notified()).await.expect("resource preparation completes");
eprintln!(
"requests through prepare={:?}",
origin.requests.lock().await[boundary..]
.iter()
.map(|request| request.lines().next().unwrap())
.collect::<Vec<_>>()
);
{
let s = session.read().await;
eprintln!(
"after prepare: binding={:?} queue={:?} states={:?}",
s.origin_account_binding,
s.segment_prefetch_queue.proxy_seqs(),
scheduled.iter().map(|(seq, _)| (*seq, &s.segments[seq].status)).collect::<Vec<_>>()
);
assert!(
scheduled.iter().all(|(seq, _)| matches!(
s.segments[seq].status,
SegmentCacheStatus::Queued { .. }
| SegmentCacheStatus::Fetching { .. }
| SegmentCacheStatus::Ready { .. }
)),
"manifest segment jobs were lost during resource preparation"
);
let current = s.origin_account_binding.as_ref().unwrap();
assert_eq!(current.generation, rebound.generation, "fresh binding stays stable into media work");
assert_eq!(current.pinned_at_ms, rebound.pinned_at_ms);
assert_eq!(
s.activity.last_delivered_media_at_ms, delivered_before_prepare,
"manifest fetch is not media delivery"
);
}
barrier.proceed.notify_one();
// No demand fetch or test-side requeue: every original job must progress autonomously.
tokio::time::timeout(WAIT, async {
loop {
let s = session.read().await;
if scheduled.iter().all(|(seq, _)| matches!(s.segments[seq].status, SegmentCacheStatus::Ready { .. })) {
break;
}
drop(s);
tokio::task::yield_now().await;
}
})
.await
.expect("all originally scheduled resources reach Ready");
let manifest = tokio::time::timeout(WAIT, opening).await.unwrap().unwrap();
let requests = origin.requests.lock().await[boundary..]
.iter()
.map(|request| request.lines().next().unwrap().to_string())
.collect::<Vec<_>>();
eprintln!("{mode:?} cycle={cycle} origin requests={requests:?}");
for (_, url) in &scheduled {
let path = reqwest::Url::parse(url).unwrap().path().to_string();
assert!(
requests.iter().any(|line| line == &format!("GET {path} HTTP/1.1")),
"scheduled origin request: {path}"
);
}
assert!(requests.iter().all(|line| !line.contains("account-user")), "B remains unused");
deliver_origin_segment(&app, &session, &manifest, current_head).await;
assert_eq!(session.read().await.origin_account_binding.as_ref().unwrap().generation, rebound.generation);
}
pause_playback(&app, &session, &lease).await;
assert!(origin.requests.lock().await.iter().all(|request| !request.contains("account-user")));
assert_eq!(app.active_provider.get_provider_connections_count(), 0, "no provider slot leak after repeated pauses");
}
#[tokio::test]
async fn hls_same_account_resume_keeps_scheduled_segments() {
same_account_resume(HlsStartupMode::Optimistic, 2).await;
}
#[tokio::test]
async fn hls_same_account_resume_keeps_scheduled_segments_conservative() {
same_account_resume(HlsStartupMode::Conservative, 2).await;
}
@@ -346,6 +346,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn test_segment_entry(
host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX),
},
proxy_seq,
media_start_ms: proxy_seq * 4_000,
duration_ms: 4_000,
proxy_file_ext: "ts".to_string(),
content_type: "video/mp2t".to_string(),
@@ -94,6 +94,8 @@ pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body_rang
SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms },
);
entry.duration_ms = 10_000;
entry.media_start_ms = proxy_seq * 10_000;
session.next_media_start_ms = session.next_media_start_ms.max(entry.media_start_ms + entry.duration_ms);
session.segments.insert(proxy_seq, entry);
}
session.advance_media_readiness_generation();
@@ -225,6 +225,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_runtime_policy_t
HlsLeasePlaybackMode::Live => "live",
HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail",
HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable",
HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response",
HlsLeasePlaybackMode::Ended => "ended",
});
panic!(
@@ -265,7 +265,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn hls_session_last_media_at
pub(in crate::api::endpoints::hls_api::tests) async fn assert_no_hls_cache_stream_registered(
app_state: &Arc<AppState>,
) {
assert!(app_state.active_users.active_streams().await.is_empty());
assert_eq!(app_state.active_users.active_streams().await.len(), 0);
}
pub(in crate::api::endpoints::hls_api::tests) async fn access_lease_session_token(
@@ -230,11 +230,16 @@ pub(in crate::api::endpoints::hls_api::tests) async fn run_recovery_outage(
let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence");
assert!(episode.full_burst_completed);
assert_eq!(episode.completed_burst_candidates, plan.total_candidates());
last_episode_generation = Some(episode.generation.0);
if let Some(generation) = last_episode_generation {
assert_eq!(episode.generation.0, generation, "unchanged failures share the initial full-burst receipt");
} else {
last_episode_generation = Some(episode.generation.0);
}
}
assert!(
fixture.origin.manifest_request_count().saturating_sub(requests_before)
>= plan.total_candidates().saturating_mul(3)
assert_eq!(
fixture.origin.manifest_request_count().saturating_sub(requests_before),
plan.total_candidates() + fixture.refresh.retry_policy.attempt_count().saturating_sub(1) + 2,
"one configured full burst, its bounded reduced retries, then one sample per due refresh"
);
let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0);
let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await;
@@ -794,9 +799,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr
session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure;
let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1);
let commit_guard_seq = transition_buffer_seq.saturating_add(1);
let base_tail = session.segments.get(&fixture.base_manifest.last_proxy_seq).expect("live base tail");
let mut media_start_ms = base_tail.media_start_ms.saturating_add(base_tail.duration_ms);
for segment in
session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq)
{
// Shortened test buffers must remain contiguous on the session media clock.
segment.media_start_ms = media_start_ms;
if segment.proxy_seq == transition_buffer_seq {
segment.duration_ms = 1_000;
} else if segment.proxy_seq == commit_guard_seq {
@@ -804,6 +813,7 @@ pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pr
} else {
segment.status = SegmentCacheStatus::Expired;
}
media_start_ms = media_start_ms.saturating_add(segment.duration_ms);
}
session.advance_media_readiness_generation();
}
@@ -915,7 +925,11 @@ pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_
.playback_cursor
.first_segment_completed_at_ms
.expect("measured lease playback start")
.saturating_add(fixture.base_manifest.playlist_duration_ms);
// Enter the acquisition window while the live base and its key are still usable.
.saturating_add(
fixture.base_manifest.playlist_duration_ms.saturating_sub(fixture.base_manifest.target_duration_ms),
);
let first = commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.app_state.hls_ctx(),
&fixture.session,
@@ -1309,6 +1323,13 @@ pub(in crate::api::endpoints::hls_api::tests) async fn assert_publication_late_t
assert!(body.contains("#EXT-X-ENDLIST"));
assert!(body.contains("/terminal/"));
}
HlsLeasePlaybackMode::TerminalCustomResponse(_) => {
assert_eq!(response.status(), StatusCode::OK);
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest utf8");
assert!(body.contains("#EXT-X-ENDLIST"));
assert!(body.contains("/cvs/hls/"));
assert!(!body.contains("/hls/shared/live/"));
}
HlsLeasePlaybackMode::TerminalUnavailable { .. } => {
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
}
@@ -1352,6 +1373,7 @@ pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_refresh_reque
transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(),
manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed,
fresh_manifest_requirement_generation: None,
refresh_identity: None,
acceptance_directive: HlsManifestAcceptanceDirective::none(),
access_lease_id,
disabled_headers: None,
@@ -86,7 +86,7 @@ async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path()
}
#[tokio::test]
async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() {
async fn commits_prepared_terminal_tail_once_before_splice_deadline_after_exhausted_recovery() {
let fixture = prepared_terminal_cutover_fixture().await;
let directive = terminal_cutover_acceptance_directive(&fixture).await;
let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await;
@@ -96,7 +96,7 @@ async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() {
}
#[tokio::test]
async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() {
async fn warm_fmp4_without_ready_reserve_uses_custom_response_after_missed_splice_deadline() {
let temp_dir = tempfile::tempdir().expect("tempdir");
let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)));
enable_channel_unavailable_custom_response(&app_state);
@@ -147,16 +147,30 @@ async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_s
let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.status(), StatusCode::OK);
assert!(!response.headers().contains_key(header::LOCATION));
assert!(response_body(response).await.is_empty());
let body = String::from_utf8(response_body(response).await.to_vec()).expect("late custom manifest");
assert!(body.ends_with("#EXT-X-ENDLIST\n"));
assert!(body.contains("/cvs/hls/"));
assert!(!body.contains("#EXT-X-MAP"));
assert!(!body.contains("/hls/shared/live/"));
let lease = app_state
.hls
.proxy
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
.await
.expect("failed-closed lease snapshot");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
.expect("late custom lease snapshot");
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await;
assert_eq!(repeated.status(), StatusCode::OK);
assert_eq!(response_body(repeated).await, body.as_bytes());
let repeated_lease = app_state
.hls
.proxy
.access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis())
.await
.expect("stable terminal custom lease");
assert_eq!(repeated_lease.playback_mode, lease.playback_mode);
let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0);
let terminal_response = get_response(app_state, &terminal_uri, None).await;
assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND);
@@ -44,6 +44,7 @@ pub(super) async fn hls_manifest_terminal_preflight(
) -> HlsManifestTerminalPreflight {
match &lease.playback_mode {
HlsLeasePlaybackMode::TerminalTail(_)
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
| HlsLeasePlaybackMode::Ended => return HlsManifestTerminalPreflight::ServeCommittedPlayback,
HlsLeasePlaybackMode::Live => {}
@@ -183,6 +184,13 @@ pub(super) fn hls_terminal_playback_response(
};
Some(hls_response(body.to_owned()))
}
HlsLeasePlaybackMode::TerminalCustomResponse(plan) => {
Some(if plan.matches_route(proxy_session_id, access_lease_id) {
hls_response(plan.manifest_body().to_owned())
} else {
StatusCode::NOT_FOUND.into_response()
})
}
HlsLeasePlaybackMode::TerminalUnavailable { .. } => Some(StatusCode::SERVICE_UNAVAILABLE.into_response()),
HlsLeasePlaybackMode::Live => None,
HlsLeasePlaybackMode::Ended => Some(StatusCode::NOT_FOUND.into_response()),
+12 -12
View File
@@ -1186,11 +1186,11 @@ mod tests {
assert_eq!(api_proxy.server.len(), 1);
assert_eq!(api_proxy.server[0].name, "main");
assert!(api_proxy.server[0].protocol.is_empty());
assert!(api_proxy.server[0].host.is_empty());
assert_eq!(api_proxy.server[0].protocol.len(), 0);
assert_eq!(api_proxy.server[0].host.len(), 0);
assert!(api_proxy.server[0].port.is_none());
assert!(api_proxy.server[0].timezone.is_empty());
assert!(api_proxy.server[0].message.is_empty());
assert_eq!(api_proxy.server[0].timezone.len(), 0);
assert_eq!(api_proxy.server[0].message.len(), 0);
assert!(api_proxy.server[0].path.is_none());
assert_eq!(api_proxy.user.len(), 1);
assert!(!api_proxy.use_user_db);
@@ -1206,8 +1206,8 @@ mod tests {
assert_eq!(api_proxy.server.len(), 1);
assert_eq!(api_proxy.server[0].name, "main");
assert!(api_proxy.server[0].host.is_empty());
assert!(api_proxy.user.is_empty());
assert_eq!(api_proxy.server[0].host.len(), 0);
assert_eq!(api_proxy.user.len(), 0);
}
#[test]
@@ -1217,8 +1217,8 @@ mod tests {
filter_api_proxy_by_permissions(&mut api_proxy, permissions);
assert!(api_proxy.server.is_empty());
assert!(api_proxy.user.is_empty());
assert_eq!(api_proxy.server.len(), 0);
assert_eq!(api_proxy.user.len(), 0);
assert!(!api_proxy.use_user_db);
assert_eq!(api_proxy.auth_error_status, 403);
}
@@ -1270,20 +1270,20 @@ mod tests {
filter_app_config_by_permissions(&mut app_config, Some(permissions));
assert_eq!(app_config.config.storage_dir, None);
assert!(app_config.sources.inputs.is_empty());
assert_eq!(app_config.sources.inputs.len(), 0);
assert!(app_config.sources.provider.is_none());
assert!(app_config.sources.templates.is_none());
assert!(app_config.mappings.is_none());
assert!(app_config.templates.is_none());
assert_eq!(app_config.sources.sources.len(), 1);
assert!(app_config.sources.sources[0].inputs.is_empty());
assert_eq!(app_config.sources.sources[0].inputs.len(), 0);
assert_eq!(app_config.sources.sources[0].targets.len(), 1);
let target = &app_config.sources.sources[0].targets[0];
assert_eq!(target.name, "Default");
assert_eq!(target.id, 1);
assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled);
assert!(target.output.is_empty());
assert_eq!(target.output.len(), 0);
assert!(target.curation.is_none(), "identity-only access must not disclose discovery credentials");
assert!(target.filter.is_empty());
assert!(target.options.is_none());
@@ -1296,7 +1296,7 @@ mod tests {
let api_proxy = app_config.api_proxy.expect("api proxy should remain present");
assert_eq!(api_proxy.server.len(), 1);
assert_eq!(api_proxy.server[0].name, "main");
assert!(api_proxy.server[0].host.is_empty());
assert_eq!(api_proxy.server[0].host.len(), 0);
assert_eq!(api_proxy.user.len(), 1);
}
@@ -344,8 +344,8 @@ async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests(
assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]);
} else {
assert!(!queued.targets.enabled);
assert!(queued.targets.targets.is_empty());
assert!(queued.targets.target_names.is_empty());
assert_eq!(queued.targets.targets.len(), 0);
assert_eq!(queued.targets.target_names.len(), 0);
}
}
}
@@ -582,7 +582,7 @@ async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_c
assert_eq!(status.inputs[0].clusters[2].timestamp, 202);
assert_eq!(status.inputs[1].input_id, 8);
assert_eq!(status.inputs[1].last_update, None);
assert!(status.inputs[1].clusters.is_empty());
assert_eq!(status.inputs[1].clusters.len(), 0);
assert_eq!(status.inputs[2].input_id, 9);
assert_eq!(status.inputs[2].last_update, Some(404));
assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed");
@@ -50,7 +50,7 @@ async fn playlist_update_status_reload_returns_active_bus_facts_without_persisti
PlaylistUpdateState::Failure,
)));
let Json(status) = super::super::playlist_update_status(State(app)).await.unwrap();
assert!(status.active_updates.is_empty());
assert_eq!(status.active_updates.len(), 0);
assert_eq!(
std::fs::read(path.join("status.json")).unwrap(),
before,
@@ -747,7 +747,7 @@ fn m3u_timeshift_resolver_returns_resolved_url_for_flussonic_template() {
.expect("M3U input handled by bridge");
assert_eq!(resolved.url, "http://provider.example/channel/video-1704067200-3600.m3u8");
assert!(!resolved.discriminator.is_empty());
assert_ne!(resolved.discriminator.len(), 0);
}
#[test]
@@ -0,0 +1,353 @@
//! Real-process regression: an origin account publication must reach every `WebUI` socket immediately.
use axum::{
extract::{OriginalUri, State},
response::IntoResponse,
Router,
};
use futures::{SinkExt, StreamExt};
use shared::model::{ActiveUserConnectionChange, ProtocolMessage, SourcesConfigDto, StreamInfo, PROTOCOL_VERSION};
use std::{
fmt::Write,
path::{Path, PathBuf},
process::Stdio,
sync::{
atomic::{AtomicUsize, Ordering},
Arc,
},
time::Duration,
};
use tokio::{
net::{TcpListener, TcpStream},
sync::Notify,
};
use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
type Socket = WebSocketStream<MaybeTlsStream<TcpStream>>;
#[derive(Clone)]
struct Origin {
catalog: String,
manifests: Arc<AtomicUsize>,
segments: Arc<AtomicUsize>,
proceed: Arc<Notify>,
}
async fn origin(State(state): State<Origin>, OriginalUri(uri): OriginalUri) -> axum::response::Response {
let path = uri.path();
if path == "/__continue" {
state.proceed.notify_one();
return "continued".into_response();
}
if path.ends_with("catalog.m3u") {
return ([("content-type", "audio/x-mpegurl")], state.catalog).into_response();
}
if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("m3u8")) {
state.manifests.fetch_add(1, Ordering::AcqRel);
return ([("content-type", "application/vnd.apple.mpegurl")], "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4,\n100.ts\n#EXTINF:4,\n101.ts\n#EXTINF:4,\n102.ts\n#EXTINF:4,\n103.ts\n#EXTINF:4,\n104.ts\n#EXTINF:4,\n105.ts\n").into_response();
}
if Path::new(path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) {
state.segments.fetch_add(1, Ordering::AcqRel);
return (
[("content-type", "video/mp2t")],
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts"))
.as_slice(),
)
.into_response();
}
axum::http::StatusCode::NOT_FOUND.into_response()
}
struct Fixture {
directory: tempfile::TempDir,
base: String,
source: PathBuf,
process: tokio::process::Child,
origins: Vec<tokio::task::JoinHandle<()>>,
b: Origin,
proceed: Arc<Notify>,
control: String,
}
impl Drop for Fixture {
fn drop(&mut self) {
let _ = self.process.start_kill();
for origin in &self.origins {
origin.abort();
}
}
}
impl Fixture {
async fn start() -> Self {
let a = TcpListener::bind("127.0.0.1:0").await.unwrap();
let b = TcpListener::bind("127.0.0.1:0").await.unwrap();
let a_addr = format!("localhost:{}", a.local_addr().unwrap().port());
let b_addr = b.local_addr().unwrap();
let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = reserved.local_addr().unwrap().port();
drop(reserved);
let directory = tempfile::tempdir().unwrap();
let root = directory.path();
for name in ["web", "storage", "backup"] {
std::fs::create_dir(root.join(name)).unwrap();
}
let frontend = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend/dist");
let web = if frontend.join("index.html").exists() { frontend } else { root.join("web") };
let config = format!("api:\n host: 127.0.0.1\n port: {port}\n web_root: '{}'\nworking_dir: '{}'\nbackup_dir: '{}'\nupdate_on_boot: true\nweb_ui:\n enabled: true\n user_ui_enabled: false\nlog:\n log_level: debug\nreverse_proxy:\n rewrite_secret: 0123456789abcdef0123456789abcdef\n stream:\n hls_session_ttl_secs: 120\n hls_cache:\n cache_path: '{}'\n", web.display(), root.join("storage").display(), root.join("backup").display(), root.join("cache").display());
let source_text = format!("inputs:\n - name: account-a\n type: m3u\n url: http://{a_addr}/catalog.m3u\n max_connections: 8\n aliases:\n - name: account-b\n url: http://{b_addr}/catalog.m3u\n max_connections: 8\nsources:\n - inputs: [account-a]\n targets:\n - name: handover\n output:\n - type: m3u\n options:\n share_live_streams:\n hls: true\n");
let mut users = format!("server:\n - name: local\n protocol: http\n host: 127.0.0.1\n port: '{port}'\n timezone: UTC\n message: Handover test\nuser:\n - target: handover\n credentials:\n");
for user in ["alice", "bob", "late"] {
write!(users, " - username: {user}\n password: test-password\n proxy: reverse\n server: local\n status: Active\n").unwrap();
}
let config_path = root.join("config.yml");
let source = root.join("source.yml");
let proxy = root.join("api-proxy.yml");
std::fs::write(&config_path, config).unwrap();
std::fs::write(&source, source_text).unwrap();
std::fs::write(&proxy, users).unwrap();
let proceed = Arc::new(Notify::new());
let catalog = format!("#EXTM3U\n#EXTINF:-1 tvg-id=\"handover\" group-title=\"Test\",Handover Channel\nhttp://{a_addr}/live/index.m3u8\n");
let state = |catalog: String| Origin {
catalog,
manifests: Arc::new(AtomicUsize::new(0)),
segments: Arc::new(AtomicUsize::new(0)),
proceed: proceed.clone(),
};
let a_state = state(catalog.clone());
let b_state = state(catalog.replace(&a_addr, &b_addr.to_string()));
let mut origins = Vec::new();
for (listener, state) in [(a, a_state), (b, b_state.clone())] {
origins.push(tokio::spawn(async move {
axum::serve(listener, Router::new().fallback(origin).with_state(state)).await.unwrap();
}));
}
let log = std::fs::File::create(root.join("sut.log")).unwrap();
let process = tokio::process::Command::new(env!("CARGO_BIN_EXE_tuliprox"))
.args(["-s", "-c"])
.arg(config_path)
.arg("-i")
.arg(&source)
.arg("-a")
.arg(proxy)
.current_dir(root)
.stdout(Stdio::from(log.try_clone().unwrap()))
.stderr(Stdio::from(log))
.kill_on_drop(true)
.spawn()
.unwrap();
Self {
directory,
base: format!("http://127.0.0.1:{port}"),
source,
process,
origins,
b: b_state,
proceed,
control: format!("http://{a_addr}/__continue"),
}
}
async fn playback(&mut self, client: &reqwest::Client, username: &str) -> String {
let url = format!("{}/m3u?username={username}&password=test-password", self.base);
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
loop {
assert!(
self.process.try_wait().unwrap().is_none(),
"SUT exited: {}",
std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap()
);
if let Ok(response) = client.get(&url).send().await {
if let Ok(body) = response.text().await {
if let Some(url) = body.lines().find(|line| line.starts_with("http")) {
return url.to_string();
}
}
}
assert!(
tokio::time::Instant::now() < deadline,
"playlist unavailable: {}",
std::fs::read_to_string(self.directory.path().join("sut.log")).unwrap()
);
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
async fn wait_for_browser_handover(&self, viewer_clients: &[reqwest::Client], viewer_media: &[reqwest::Url]) {
eprintln!(
"BROWSER_READY web={} control={} fixture={}",
self.base,
self.control,
self.directory.path().display()
);
// Keep the two manual test players running while the browser loads. Stop all
// media requests before publishing B; the browser must then update from events alone.
let mut playback = tokio::time::interval(Duration::from_secs(1));
loop {
tokio::select! {
() = self.proceed.notified() => break,
_ = playback.tick() => {
for (client, media) in viewer_clients.iter().zip(viewer_media) {
let response = client.get(media.clone()).send().await.unwrap();
assert!(response.status().is_success());
response.bytes().await.unwrap();
}
}
}
}
}
}
async fn socket(base: &str) -> Socket {
let (mut socket, _) = connect_async(format!("{}/ws", base.replace("http://", "ws://"))).await.unwrap();
socket.send(Message::Binary(ProtocolMessage::Version(PROTOCOL_VERSION).to_bytes().unwrap())).await.unwrap();
assert!(
matches!(socket.next().await.unwrap().unwrap(), Message::Binary(bytes) if bytes.as_ref() == [PROTOCOL_VERSION])
);
socket
}
async fn frame(socket: &mut Socket) -> ProtocolMessage {
loop {
let next = tokio::time::timeout(Duration::from_secs(15), socket.next())
.await
.expect("WebUI frame arrives without polling")
.unwrap()
.unwrap();
if let Message::Binary(bytes) = next {
return ProtocolMessage::from_bytes(bytes).unwrap();
}
}
}
async fn snapshot(socket: &mut Socket) -> Vec<StreamInfo> {
socket.send(Message::Binary(ProtocolMessage::StatusRequest(String::new()).to_bytes().unwrap())).await.unwrap();
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
loop {
if let ProtocolMessage::StatusResponse(status) =
tokio::time::timeout_at(deadline, frame(socket)).await.expect("full status snapshot arrives")
{
return status.active_user_streams;
}
}
}
async fn changed_viewers(socket: &mut Socket, before: &[StreamInfo]) {
let mut seen = std::collections::HashSet::new();
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
while seen.len() < before.len() {
match tokio::time::timeout_at(deadline, frame(socket)).await.expect("all viewers receive Updated frames") {
ProtocolMessage::ActiveUserResponse(ActiveUserConnectionChange::Updated(row))
if row.provider.as_ref() == "account-b" =>
{
let mut expected = before
.iter()
.find(|old| old.uid == row.uid && old.addr == row.addr)
.expect("same viewer row")
.clone();
expected.provider = Arc::from("account-b");
assert_eq!(row, expected, "handover changes only the displayed provider");
seen.insert(row.uid);
}
ProtocolMessage::StatusResponse(_) => panic!("handover must arrive as incremental Updated frames"),
_ => {}
}
}
}
#[tokio::test]
async fn hls_origin_account_handover() {
let mut fixture = Fixture::start().await;
let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap();
let mut canonical = Vec::new();
let mut viewer_clients = Vec::new();
let mut viewer_media = Vec::new();
for viewer in ["alice", "bob"] {
let client = reqwest::Client::builder().timeout(Duration::from_secs(15)).build().unwrap();
let mut entry = fixture.playback(&client, viewer).await;
let mut manifest = None;
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
while tokio::time::Instant::now() < deadline {
let response = client.get(&entry).send().await.unwrap();
if response.status().is_success() {
let base = response.url().clone();
let body = response.text().await.unwrap();
let segments = body
.lines()
.filter(|line| !line.starts_with('#') && !line.is_empty())
.map(|line| base.join(line).unwrap())
.collect::<Vec<_>>();
if let Some(segment) = segments.first() {
if body.contains("#EXT-X-STREAM-INF:") {
entry = segment.to_string();
continue;
}
let media = client.get(segment.clone()).send().await.unwrap();
assert!(media.status().is_success());
assert!(
!media.bytes().await.unwrap().is_empty(),
"viewer actually consumes media before the segment gap"
);
viewer_media.push(segment.clone());
manifest = Some(base);
break;
}
}
tokio::task::yield_now().await;
}
canonical.push(manifest.expect("viewer has a canonical playable manifest"));
viewer_clients.push(client);
}
let mut ui_a = socket(&fixture.base).await;
let mut ui_b = socket(&fixture.base).await;
let mut before = snapshot(&mut ui_a).await;
assert_eq!(
before.len(),
2,
"canonical={canonical:?} SUT log: {}",
std::fs::read_to_string(fixture.directory.path().join("sut.log")).unwrap()
);
assert!(before.iter().all(|row| row.provider.as_ref() == "account-a"));
let mut before_b = snapshot(&mut ui_b).await;
assert_eq!(before_b.len(), 2);
let manual = std::env::var_os("TULIPROX_HANDOVER_BROWSER_CHECK").is_some();
if manual {
fixture.wait_for_browser_handover(&viewer_clients, &viewer_media).await;
// Playback above may advance timestamps; capture the exact pre-publication rows.
before = snapshot(&mut ui_a).await;
before_b = snapshot(&mut ui_b).await;
assert_eq!(before.len(), 2);
assert_eq!(before_b.len(), 2);
}
let mut sources: SourcesConfigDto =
serde_saphyr::from_str(&std::fs::read_to_string(&fixture.source).unwrap()).unwrap();
sources.inputs[0].account_disabled = true;
let config = client.get(format!("{}/api/v1/config", fixture.base)).send().await.unwrap();
let revision =
config.headers().get(shared::utils::HEADER_CONFIG_SOURCES_REVISION).expect("source revision").clone();
let response = client
.post(format!("{}/api/v1/config/sources", fixture.base))
.header(reqwest::header::IF_MATCH, revision)
.json(&sources)
.send()
.await
.unwrap();
assert!(response.status().is_success(), "source reload: {}", response.text().await.unwrap());
// Only a manifest refresh triggers the handover; neither viewer requests another segment or status.
let response = client.get(canonical[0].clone()).send().await.unwrap();
assert!(response.status().is_success() || response.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE);
changed_viewers(&mut ui_a, &before).await;
changed_viewers(&mut ui_b, &before_b).await;
eprintln!(
"HANDOVER_EVENTS_DELIVERED replacement_manifest_requests={} replacement_segment_requests={}",
fixture.b.manifests.load(Ordering::Acquire),
fixture.b.segments.load(Ordering::Acquire)
);
let after = snapshot(&mut ui_a).await;
assert_eq!(after.len(), 2);
assert!(after.iter().all(|row| row.provider.as_ref() == "account-b"));
if manual {
eprintln!("BROWSER_SWITCHED web={} control={}", fixture.base, fixture.control);
fixture.proceed.notified().await;
}
fixture.process.kill().await.unwrap();
}
@@ -793,7 +793,7 @@ mod tests {
let response = local_response(StatusCode::OK, &[], encoded_binary.clone()).await;
let mut untouched =
decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly).await.expect("identity setup");
assert!(untouched.decoded_from.is_empty());
assert_eq!(untouched.decoded_from.len(), 0);
assert_eq!(read_to_end_limited(&mut untouched.body, 1024).await.expect("untouched body"), encoded_binary);
}
@@ -806,7 +806,7 @@ mod tests {
.await
.expect("identity setup");
assert!(decoded.decoded_from.is_empty());
assert_eq!(decoded.decoded_from.len(), 0);
assert_eq!(decoded.content_coding_observation(), None);
assert!(!decoded.headers.contains_key(CONTENT_ENCODING));
assert_eq!(read_to_end_limited(&mut decoded.body, 1024).await.expect("identity body"), encoded);
+2 -2
View File
@@ -52,11 +52,11 @@ pub enum ResourceDestination {
}
/// How long a verdict that allows a URL to be handed to a client is remembered.
const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_secs(60);
const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_mins(1);
/// How long a verdict that keeps a destination away from clients is remembered. Both classes fail
/// closed, so they can be cached longer than the one that decides exposure.
const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_secs(300);
const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_mins(5);
/// How long a name that produced no answer within the resolution budget is remembered.
///
@@ -108,7 +108,7 @@ async fn atomic_download_temp_files_are_unique_and_use_target_directory() {
drop(second_output);
drop(first_temp);
drop(second_temp);
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[test]
@@ -181,7 +181,7 @@ async fn local_epg_file_respects_max_download_bytes() {
assert!(err.to_string().contains("exceeds configured limit"));
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[tokio::test]
@@ -213,7 +213,7 @@ async fn file_url_epg_source_respects_streamed_size_limit() {
assert!(err.to_string().contains("exceeds configured limit"));
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[tokio::test]
@@ -243,7 +243,7 @@ async fn local_epg_replace_error_cleans_temp_file() {
.expect_err("replacing a directory should fail");
assert!(persist.is_dir());
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[tokio::test]
@@ -273,7 +273,7 @@ async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() {
.expect_err("reading a directory as an EPG file should fail");
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[tokio::test]
@@ -301,7 +301,7 @@ async fn unsupported_epg_url_scheme_is_rejected_without_network_access() {
assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'"));
assert!(!persist.exists());
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
#[cfg(windows)]
@@ -370,7 +370,7 @@ async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() {
assert!(err.to_string().contains("exceeds configured limit"));
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
server_handle.abort();
}
@@ -409,7 +409,7 @@ async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file()
.expect_err("truncated response body should fail");
assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
server_handle.abort();
}
@@ -466,7 +466,7 @@ async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomic
assert_eq!(accepted.load(Ordering::SeqCst), 2);
assert_eq!(max_active.load(Ordering::SeqCst), 1);
assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n");
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
server_handle.abort();
}
@@ -37,5 +37,5 @@ async fn file_url_epg_source_is_copied_to_persist_path() {
tokio::fs::read_to_string(&persist).await.expect("persisted content"),
"BEGIN:VCALENDAR\nEND:VCALENDAR\n"
);
assert!(atomic_download_temp_files(dir.path()).await.is_empty());
assert_eq!(atomic_download_temp_files(dir.path()).await.len(), 0);
}
+1 -1
View File
@@ -156,7 +156,7 @@ async fn empty_and_no_match_are_complete_but_disabled_sources_are_not_configured
let CurationRunOutcome::Complete(result) = run(&config(None), &catalog(), &server).await else {
panic!("valid empty or no-match is complete")
};
assert!(result.memberships.is_empty());
assert_eq!(result.memberships.len(), 0);
assert_eq!(result.selectors.len(), 2);
}
let server = TestServer::new(http_response(200, PAGE)).await;
+2 -2
View File
@@ -699,7 +699,7 @@ mod tests {
panic!("an empty source is complete")
};
assert_eq!(reference_count, 0);
assert!(memberships.is_empty());
assert_eq!(memberships.len(), 0);
let no_match = vec![reference(CurationMediaKind::Movie, "Missing", None, Some(8), Some(1))];
let SelectorOutcome::Complete { reference_count, memberships, .. } =
@@ -708,7 +708,7 @@ mod tests {
panic!("a complete source with no local match is complete")
};
assert_eq!(reference_count, 1);
assert!(memberships.is_empty());
assert_eq!(memberships.len(), 0);
let matched = vec![reference(CurationMediaKind::Movie, "Matched", None, Some(7), Some(1))];
let SelectorOutcome::Complete { reference_count, memberships, .. } =
+1 -1
View File
@@ -36,7 +36,7 @@ impl TestServer {
}
pub(crate) async fn serve(responses: Vec<Vec<u8>>, delay: Duration, send_headers: bool) -> Self {
assert!(!responses.is_empty());
assert_ne!(responses.len(), 0);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}/", listener.local_addr().unwrap());
let requests = Arc::new(Mutex::new(Vec::new()));
+3 -3
View File
@@ -136,7 +136,7 @@ async fn tmdb_empty_is_valid_only_at_the_start_with_coherent_totals() {
for pages in [0, 1] {
let body = json!({"page":1,"total_pages":pages,"total_results":0,"results":[]}).to_string();
let server = TestServer::new(http_response(200, &body)).await;
assert!(client(&server).fetch(&selector(100)).await.unwrap().is_empty());
assert_eq!(client(&server).fetch(&selector(100)).await.unwrap().len(), 0);
assert_eq!(server.requests.lock().unwrap().len(), 1);
}
for body in [
@@ -222,7 +222,7 @@ async fn tmdb_all_routes_preserve_headers_with_no_query_token_or_details() {
let mut selector = selector(100);
selector.kind = kind;
selector.time_window = window;
assert!(client.fetch(&selector).await.unwrap().is_empty());
assert_eq!(client.fetch(&selector).await.unwrap().len(), 0);
let requests = server.requests.lock().unwrap();
assert_eq!(requests.len(), 1);
assert!(
@@ -285,7 +285,7 @@ async fn tmdb_identity_chunked_and_gzip_exact_and_plus_one_count_decoded_bytes()
let mut budget = client.batch_budget();
let result = client.trending(&selector(100), &mut budget).await;
if length == BODY_LIMIT {
assert!(result.unwrap().is_empty());
assert_eq!(result.unwrap().len(), 0);
} else {
assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit);
}
+1 -1
View File
@@ -343,7 +343,7 @@ mod tests {
assert_eq!(evaluation.selectors.len(), 1);
assert_eq!(evaluation.selectors[0].reference_count, 0);
assert_eq!(evaluation.selectors[0].membership_count, 0);
assert!(evaluation.memberships.is_empty());
assert_eq!(evaluation.memberships.len(), 0);
server.abort();
}
@@ -320,7 +320,7 @@ mod tests {
None,
);
assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict);
assert!(preview.overlap_segments.is_empty());
assert_eq!(preview.overlap_segments.len(), 0);
}
#[test]
@@ -706,7 +706,7 @@ mod tests {
let dir = TempDir::new().expect("tempdir");
let (queue, shared_file, target) = delete_one_of_two_entries_sharing(&dir).await;
assert!(target.still_referenced, "Bob still holds this file");
assert!(target.paths_to_unlink().is_empty());
assert_eq!(target.paths_to_unlink().len(), 0);
let unlinked = execute_deletion_target(&target).await.expect("execute");
assert_eq!(unlinked, None, "nothing was unlinked");
assert!(shared_file.exists(), "Bob's recording must survive Alice's deletion");
@@ -233,7 +233,7 @@ fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again()
};
let promoted = promote_from_queue(&mut candidate);
assert!(promoted.is_none(), "nothing needs to run");
assert!(candidate.active.is_empty());
assert_eq!(candidate.active.len(), 0);
assert!(candidate.queue.is_empty(), "bob left the queue");
let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed");
assert_eq!(bob.state, RecordingTaskState::Completed);
@@ -355,7 +355,7 @@ mod tests {
let mut t = completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1);
t.recording.completed_at = None;
let out = compute_candidates(&[t], &config, 1_000_000_000);
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -593,6 +593,6 @@ mod tests {
let config = RetentionConfig::default();
let tasks = vec![completed("a", RecordingOwner::User(UserId::from("web:alice")), Some("c1"), Some("Alpha"), 1)];
let out = compute_candidates(&tasks, &config, 0);
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
}
@@ -442,7 +442,7 @@ mod tests {
let key = occurrence_key("r1", &source(), "ch-1", 1_000, "t:my show");
let tasks = vec![task("r1", &key)];
let out = plan_materializations(&rules, &programmes, &tasks, &TombstoneSet::default(), 0, Some(10_000));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -450,7 +450,7 @@ mod tests {
let rules = vec![new_episode_rule()];
let programmes = vec![epg_programme("My Show", -1, 100)];
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -458,7 +458,7 @@ mod tests {
let rules = vec![new_episode_rule()];
let programmes = vec![epg_programme("My Show", 5_000, 6_000)];
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(2_000));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -468,7 +468,7 @@ mod tests {
p.previously_shown = true;
let programmes = vec![p];
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -478,7 +478,7 @@ mod tests {
let rules = vec![r];
let programmes = vec![epg_programme("My Show", 1_000, 2_000)];
let out = plan_materializations(&rules, &programmes, &[], &TombstoneSet::default(), 0, Some(10_000));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -502,7 +502,7 @@ mod tests {
// Set the EPG horizon to 1 hour — the next Sunday is
// days away.
let out = plan_materializations(&rules, &[], &[], &TombstoneSet::default(), now, Some(3_600));
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
#[test]
@@ -514,6 +514,6 @@ mod tests {
let out = plan_materializations(&[rule], &[], &[], &TombstoneSet::default(), 0, None);
assert!(out.is_empty());
assert_eq!(out.len(), 0);
}
}
+10 -5
View File
@@ -405,7 +405,7 @@ pub async fn recovery_decision_for(final_path: &Path, partial: &Path) -> Recover
#[cfg(test)]
mod tests {
#[cfg(unix)]
#[cfg(target_os = "linux")]
use super::spawn_retrying_busy_executable;
use super::{
build_recording_args, classify_ffmpeg_failure, recording_partial_path,
@@ -778,7 +778,8 @@ mod tests {
let _ = fs::remove_dir_all(&recording.file_dir);
}
#[cfg(unix)]
// Linux rejects execution while a writer is open; macOS permits it.
#[cfg(target_os = "linux")]
#[tokio::test]
async fn spawn_retries_while_executable_is_open_for_writing() {
let script = fake_ffmpeg_script("busy", "#!/bin/sh\nexit 0\n");
@@ -970,8 +971,7 @@ mod tests {
};
assert!(generated.status.success(), "{}", String::from_utf8_lossy(&generated.stderr));
let bytes = tokio::fs::read(&input).await?;
// This address is outside the usual environment's localhost exclusion.
let listener = tokio::net::TcpListener::bind("127.0.0.2:0").await?;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let url =
reqwest::Url::parse(&format!("http://{}/api/v1/playlist/recording/secret/live", listener.local_addr()?))?;
let (request_tx, request_rx) = tokio::sync::oneshot::channel();
@@ -991,7 +991,12 @@ mod tests {
Ok::<_, std::io::Error>(())
});
let wrapper = directory.path().join("ffmpeg-with-proxy");
tokio::fs::write(&wrapper, "#!/bin/sh\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n").await?;
// Explicitly remove proxy exclusions so localhost still exercises the bypass.
tokio::fs::write(
&wrapper,
"#!/bin/sh\nunset no_proxy NO_PROXY\nexport http_proxy=http://127.0.0.1:9\nexec ffmpeg \"$@\"\n",
)
.await?;
fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755))?;
let mut recording = make_recording(chrono::Utc::now().timestamp(), 60);
recording.url = url;
+2 -2
View File
@@ -276,7 +276,7 @@ mod tests {
let queue = RecordingQueue::new();
let snapshot = recording_snapshot(&queue, &no_subject_claims(), &bare_app_config()).await;
assert_eq!(snapshot.revision.0, 0);
assert!(snapshot.tasks.is_empty());
assert_eq!(snapshot.tasks.len(), 0);
assert_eq!(snapshot.quota, RecordingQuotaSummaryDto::default(), "a session with no subject is charged nothing");
}
@@ -285,7 +285,7 @@ mod tests {
let queue = RecordingQueue::new();
let snapshot = recording_snapshot(&queue, &no_read_claims(), &bare_app_config()).await;
assert_eq!(snapshot.revision.0, 0);
assert!(snapshot.tasks.is_empty());
assert_eq!(snapshot.tasks.len(), 0);
}
/// Every internal fact a recording carries that a client must never
@@ -0,0 +1,46 @@
//! Publish degraded origin evidence before the manifest retry loop starts waiting.
use super::reevaluation::register_terminal_reserve_reevaluation;
use crate::{
manifest_fetch::{HlsOriginManifestFetchContext, OriginManifestFetchError},
origin_progress::HlsOriginPathCondition,
post_refresh_availability::commit_post_refresh_terminal_fallback,
refresh::{HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason},
};
use std::sync::Arc;
pub(crate) async fn record_retryable_manifest_attempt_failure(
context: &HlsOriginManifestFetchContext,
expected_progress_generation: u64,
error: &OriginManifestFetchError,
now_ms: u64,
) {
let action = {
let mut session = context.session.write().await;
if session.is_gc_marked_for_removal()
|| session.origin_control.progress_generation != expected_progress_generation
{
return;
}
if matches!(error, OriginManifestFetchError::RetryableStatus(..)) {
session.origin_control.record_origin_response(now_ms);
}
// An individual soft failure must not weaken an already established hard failure or conflict.
if matches!(
session.origin_control.path_condition,
HlsOriginPathCondition::ProgressExpected | HlsOriginPathCondition::RetryableFetchFailure
) {
session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure;
}
HlsPostRefreshAvailabilityAction::Reevaluate {
reason: HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
origin_progress_generation: session.origin_control.progress_generation,
media_readiness_generation: session.activity.media_readiness_generation,
}
};
let Some(ctx) = context.availability_runtime.as_ref().and_then(crate::hls_ctx::WeakHlsCtx::upgrade) else {
return;
};
let registration = register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(&context.session)).await;
let _ = commit_post_refresh_terminal_fallback(ctx, Arc::clone(&context.session), action, registration).await;
}
+8 -1
View File
@@ -17,11 +17,18 @@ use super::{
HlsLiveReserveDeadline, HlsOriginProgressDecision, ProxySessionId,
};
mod manifest_failure;
mod recovery_pressure;
mod reevaluation;
mod terminal_cutover;
mod terminal_pending_owner;
pub(crate) use self::terminal_cutover::commit_terminal_tail_if_lease_reserve_requires_cutover_detailed;
pub(crate) use self::{
manifest_failure::record_retryable_manifest_attempt_failure,
terminal_cutover::{
commit_terminal_tail_if_lease_reserve_requires_cutover_detailed, terminal_bundle_failure_compatibility,
terminal_bundle_incompatibility,
},
};
use self::{
recovery_pressure::{
evaluate_and_commit_session_recovery_pressure_in_snapshot, hls_recovery_pressure_policy,
+29 -5
View File
@@ -347,7 +347,7 @@ async fn evaluate_post_refresh_terminal_attempt(
cycle: &mut HlsAvailabilityReevaluationCycle,
now_ms: u64,
) -> HlsTerminalReevaluationDecision {
if !matches!(mode, HlsAvailabilityReevaluationMode::PostRefresh(_)) {
if matches!(mode, HlsAvailabilityReevaluationMode::RecoveryPressure) {
return HlsTerminalReevaluationDecision::Continue { terminal_evaluation_pending: false };
}
match evaluate_active_terminal_leases_for_reevaluation(ctx, session, &owner_key.proxy_session_id, now_ms).await {
@@ -439,7 +439,7 @@ async fn run_hls_availability_reevaluation_attempt(
session: &mut HlsSessionHandle,
owner_key: &mut HlsAvailabilityReevaluationOwnerKey,
ownership: &HlsAvailabilityReevaluationOwnership,
refresh_request: &mut OriginRefreshRequest,
refresh_request: &mut Option<OriginRefreshRequest>,
cycle: &mut HlsAvailabilityReevaluationCycle,
) -> HlsAvailabilityCycleDecision {
if !ownership.is_current(owner_key) {
@@ -463,6 +463,9 @@ async fn run_hls_availability_reevaluation_attempt(
HlsTerminalReevaluationDecision::RestartCycle => return HlsAvailabilityCycleDecision::RestartCycle,
HlsTerminalReevaluationDecision::Stop => return HlsAvailabilityCycleDecision::Stop,
};
let Some(refresh_request) = refresh_request.as_mut() else {
return HlsAvailabilityCycleDecision::FinishCycle;
};
let schedule = match evaluate_availability_refresh_attempt(
ctx,
session,
@@ -482,10 +485,13 @@ async fn run_hls_availability_reevaluation_attempt(
return HlsAvailabilityCycleDecision::FinishCycle;
}
let retry_from_ms = ctx.hls_proxy.now_ms();
let Some(retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else {
let Some(mut retry_at_ms) = schedule.wake_at_ms(retry_from_ms, cycle.attempts_completed, cycle.deadline_ms) else {
warn!("HLS availability reevaluation cycle stopped: reason=deadline_elapsed");
return HlsAvailabilityCycleDecision::FinishCycle;
};
if let Some(deadline) = cycle.live_reserve_deadline {
retry_at_ms = retry_at_ms.min(deadline.next_reevaluation_at_ms);
}
match wait_for_availability_owner_signal(ownership, retry_at_ms, retry_from_ms).await {
HlsAvailabilityOwnerWaitOutcome::Cancelled => {
ownership.discard_superseded(owner_key);
@@ -504,7 +510,7 @@ async fn run_hls_availability_reevaluation(
mut session: HlsSessionHandle,
mut owner_key: HlsAvailabilityReevaluationOwnerKey,
ownership: HlsAvailabilityReevaluationOwnership,
mut refresh_request: OriginRefreshRequest,
mut refresh_request: Option<OriginRefreshRequest>,
) {
'cycles: loop {
let mut cycle = HlsAvailabilityReevaluationCycle::new(ctx.hls_proxy.now_ms());
@@ -605,6 +611,24 @@ pub(super) fn register_hls_availability_reevaluation_with_mode(
let coordinator = ctx.hls_proxy.availability_reevaluations();
let registration_key = owner_key.clone();
coordinator.register(registration_key, mode, move |ownership| {
run_hls_availability_reevaluation(ctx, session, owner_key, ownership, refresh_request)
run_hls_availability_reevaluation(ctx, session, owner_key, ownership, Some(refresh_request))
})
}
pub(super) fn register_terminal_reserve_reevaluation(
ctx: HlsCtx,
session: HlsSessionHandle,
) -> futures::future::BoxFuture<'static, HlsAvailabilityReevaluationRegistration> {
Box::pin(async move {
let proxy_session_id = session.read().await.proxy_session_id.clone();
let Some(owner_key) = ctx.hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await
else {
return HlsAvailabilityReevaluationRegistration::Superseded;
};
ctx.hls_proxy.availability_reevaluations().register(
owner_key.clone(),
HlsAvailabilityReevaluationMode::TerminalReserve,
move |ownership| run_hls_availability_reevaluation(ctx, session, owner_key, ownership, None),
)
})
}
@@ -8,9 +8,9 @@
use super::{
recovery_pressure::{hls_recovery_pressure_policy, recovery_trigger_budget},
terminal_pending_owner::{
commit_prepared_terminal_unavailable_after_owner_failure, commit_terminal_unavailable,
register_ready_terminal_pending_owner, register_terminal_pending_owner, terminal_asset_revision_guard,
terminal_resolution_with_failed_closed_fallback,
commit_late_custom_response, commit_prepared_terminal_unavailable_after_owner_failure,
commit_terminal_unavailable, register_ready_terminal_pending_owner, register_terminal_pending_owner,
terminal_asset_revision_guard, terminal_resolution_with_failed_closed_fallback,
},
HlsDetailedTerminalResolution, HlsTerminalDecisionPurpose, HlsTerminalFailedClosedReason, HlsTerminalResolution,
HLS_PLAYBACK_RATE_GUARD_MILLI,
@@ -184,6 +184,7 @@ async fn resolve_terminal_cutover_before_commit_window(
&context.lease.lease_id,
&preparation,
context.now_ms,
HlsTerminalTailCompatibility::TerminalMediaNotReady,
),
context.evaluation.safe_deadline,
);
@@ -239,7 +240,7 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover(
lease: &HlsAccessLease,
now_ms: u64,
) -> HlsTerminalResolution {
commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
let detailed = commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
ctx,
session,
proxy_session_id,
@@ -247,8 +248,30 @@ pub async fn commit_terminal_tail_if_lease_reserve_requires_cutover(
now_ms,
HlsTerminalDecisionPurpose::OrdinaryCutover,
)
.await
.resolution
.await;
if detailed.live_reserve_deadline.is_some()
&& matches!(detailed.resolution, HlsTerminalResolution::LiveAllowed | HlsTerminalResolution::Reevaluate)
{
let registration =
super::reevaluation::register_terminal_reserve_reevaluation(ctx.clone(), Arc::clone(session)).await;
if matches!(
registration,
crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::CapacityExceeded
| crate::availability_reevaluation::HlsAvailabilityReevaluationRegistration::RuntimeUnavailable
) {
return commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
ctx,
session,
proxy_session_id,
lease,
now_ms,
HlsTerminalDecisionPurpose::AutonomousOwnerFailureFallback,
)
.await
.resolution;
}
}
detailed.resolution
}
pub(crate) async fn commit_terminal_tail_if_lease_reserve_requires_cutover_detailed(
@@ -341,13 +364,26 @@ async fn commit_prepared_terminal_decision(
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
) -> HlsTerminalResolution {
if !preparation.trigger.is_runtime_policy()
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
{
return commit_late_custom_response(HlsTerminalCommitContext {
ctx,
session,
proxy_session_id,
lease_id,
preparation,
now_ms,
})
.await;
}
let reason = preparation.trigger.reason();
let asset = match snapshot_hls_runtime_custom_tail_asset(ctx, reason) {
Ok(asset) => asset,
Err(compatibility) => {
let context = HlsTerminalCommitContext { ctx, session, proxy_session_id, lease_id, preparation, now_ms };
let outcome = commit_terminal_unavailable(context, None, compatibility.terminal_compatibility());
return terminal_resolution_with_failed_closed_fallback(context, outcome);
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
}
};
commit_prepared_runtime_custom_tail(ctx, session, proxy_session_id, lease_id, preparation, now_ms, asset).await
@@ -366,6 +402,11 @@ pub async fn commit_prepared_runtime_custom_tail(
if preparation.trigger.reason() != asset.reason {
return HlsTerminalResolution::Reevaluate;
}
if !preparation.trigger.is_runtime_policy()
&& now_ms >= preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch()
{
return commit_late_custom_response(context).await;
}
let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&asset);
let media_asset = asset.asset;
let static_incompatibility = if preparation.manifest_snapshot.delivery_mode
@@ -381,7 +422,7 @@ pub async fn commit_prepared_runtime_custom_tail(
};
if let Some(reason) = static_incompatibility {
let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason);
return terminal_resolution_with_failed_closed_fallback(context, outcome);
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
}
let bundle_key = prepared_terminal_bundle_key(
&media_asset,
@@ -403,7 +444,7 @@ pub async fn commit_prepared_runtime_custom_tail(
Some(expected_asset),
HlsTerminalTailCompatibility::AssetRevisionMismatch,
);
return terminal_resolution_with_failed_closed_fallback(context, outcome);
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
}
HlsPreparedTerminalBundleState::Preparing { .. } => {
return register_terminal_pending_owner(context, media_asset, expected_asset, bundle_key).await;
@@ -411,12 +452,12 @@ pub async fn commit_prepared_runtime_custom_tail(
HlsPreparedTerminalBundleState::Failed { key, reason } => {
let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason);
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
return terminal_resolution_with_failed_closed_fallback(context, outcome);
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
}
HlsPreparedTerminalBundleState::Incompatible { reason, .. } => {
let reason = terminal_bundle_incompatibility(reason);
let outcome = commit_terminal_unavailable(context, Some(expected_asset), reason);
return terminal_resolution_with_failed_closed_fallback(context, outcome);
return terminal_resolution_with_failed_closed_fallback(context, outcome).await;
}
};
if preparation.trigger.is_runtime_policy() {
@@ -426,10 +467,11 @@ pub async fn commit_prepared_runtime_custom_tail(
expected_asset,
bundle_key,
prepared_bundle,
);
)
.await;
}
let outcome = commit_ready_terminal_bundle(context, media_asset, expected_asset, bundle_key, prepared_bundle).await;
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
struct HlsReadyTerminalSplice {
@@ -624,7 +666,7 @@ pub(super) async fn commit_ready_terminal_bundle(
}
}
pub(super) fn terminal_bundle_incompatibility(
pub(crate) fn terminal_bundle_incompatibility(
reason: HlsPreparedTerminalBundleIncompatibility,
) -> HlsTerminalTailCompatibility {
match reason {
@@ -636,7 +678,7 @@ pub(super) fn terminal_bundle_incompatibility(
}
}
pub(super) fn terminal_bundle_failure_compatibility(
pub(crate) fn terminal_bundle_failure_compatibility(
actual_key: HlsPreparedTerminalBundleKey,
required_key: HlsPreparedTerminalBundleKey,
reason: HlsPreparedTerminalBundleFailure,
@@ -89,7 +89,7 @@ pub(super) async fn register_terminal_pending_owner(
ownership,
)
});
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration)
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await
}
fn terminal_pending_owner_key(
@@ -117,7 +117,7 @@ fn terminal_pending_owner_key(
})
}
pub(super) fn register_ready_terminal_pending_owner(
pub(super) async fn register_ready_terminal_pending_owner(
context: HlsTerminalCommitContext<'_>,
asset: Arc<crate::terminal_tail::HlsTerminalMediaAsset>,
expected_asset: HlsRuntimeCustomTailAssetIdentity,
@@ -150,16 +150,15 @@ pub(super) fn register_ready_terminal_pending_owner(
};
let outcome = commit_ready_terminal_bundle(task_context, asset, expected_asset, bundle_key, bundle).await;
if ownership.is_current() {
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(
task_context,
outcome,
));
observe_autonomous_terminal_resolution(
terminal_resolution_with_failed_closed_fallback(task_context, outcome).await,
);
}
});
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration)
terminal_resolution_for_pending_registration(context, expected_asset, latest_safe_commit_at_ms, registration).await
}
pub(super) fn terminal_resolution_for_pending_registration(
pub(super) async fn terminal_resolution_for_pending_registration(
context: HlsTerminalCommitContext<'_>,
expected_asset: HlsRuntimeCustomTailAssetIdentity,
latest_safe_commit_at_ms: u64,
@@ -178,7 +177,7 @@ pub(super) fn terminal_resolution_for_pending_registration(
Some(expected_asset),
HlsTerminalTailCompatibility::TerminalMediaNotReady,
);
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
}
}
@@ -193,7 +192,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
match state {
Some(HlsPreparedTerminalBundleState::Ready { bundle }) if bundle.matches_key_and_shape(bundle_key) => {
let outcome = commit_ready_terminal_bundle(context, asset, expected_asset, bundle_key, bundle).await;
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
Some(HlsPreparedTerminalBundleState::Ready { .. }) => {
let outcome = commit_terminal_unavailable(
@@ -201,12 +200,12 @@ async fn terminal_resolution_after_settled_bundle_observation(
Some(expected_asset),
HlsTerminalTailCompatibility::AssetRevisionMismatch,
);
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
Some(HlsPreparedTerminalBundleState::Failed { key, reason }) => {
let compatibility = terminal_bundle_failure_compatibility(key, bundle_key, reason);
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
Some(HlsPreparedTerminalBundleState::Incompatible { key, reason }) => {
let compatibility = if key == bundle_key {
@@ -215,7 +214,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
HlsTerminalTailCompatibility::AssetRevisionMismatch
};
let outcome = commit_terminal_unavailable(context, Some(expected_asset), compatibility);
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
Some(HlsPreparedTerminalBundleState::Preparing { .. }) | None => {
let outcome = commit_terminal_unavailable(
@@ -223,7 +222,7 @@ async fn terminal_resolution_after_settled_bundle_observation(
Some(expected_asset),
HlsTerminalTailCompatibility::TerminalMediaNotReady,
);
terminal_resolution_with_failed_closed_fallback(context, outcome)
terminal_resolution_with_failed_closed_fallback(context, outcome).await
}
}
}
@@ -330,7 +329,7 @@ pub(super) async fn run_terminal_pending_owner(
commit_terminal_unavailable(context, Some(expected_asset), compatibility)
}
};
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome));
observe_autonomous_terminal_resolution(terminal_resolution_with_failed_closed_fallback(context, outcome).await);
}
fn terminal_pending_retry_after_ms(now_ms: u64, latest_safe_commit_at_ms: u64) -> u64 {
@@ -378,7 +377,7 @@ pub(super) fn terminal_resolution_for_commit_outcome(
}
}
pub(super) fn terminal_resolution_with_failed_closed_fallback(
pub(super) async fn terminal_resolution_with_failed_closed_fallback(
context: HlsTerminalCommitContext<'_>,
outcome: HlsTerminalCommitOutcome,
) -> HlsTerminalResolution {
@@ -389,6 +388,9 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback(
if context.preparation.trigger.is_runtime_policy() {
return resolution;
}
if outcome == HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed {
return commit_late_custom_response(context).await;
}
commit_prepared_terminal_unavailable_after_owner_failure(
context.ctx,
context.session,
@@ -396,9 +398,48 @@ pub(super) fn terminal_resolution_with_failed_closed_fallback(
context.lease_id,
context.preparation,
context.now_ms,
HlsTerminalTailCompatibility::TerminalMediaNotReady,
)
}
pub(super) async fn commit_late_custom_response(context: HlsTerminalCommitContext<'_>) -> HlsTerminalResolution {
let prepared = crate::late_custom_response::prepare_late_custom_response(
context.ctx,
context.proxy_session_id,
context.lease_id,
context.preparation,
context.now_ms,
)
.await;
let (plan, expected_asset) = match prepared {
Ok(prepared) => prepared,
Err(crate::late_custom_response::HlsLateCustomResponseError::Superseded) => {
return HlsTerminalResolution::Reevaluate
}
Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) => {
return commit_prepared_terminal_unavailable_after_owner_failure(
context.ctx,
context.session,
context.proxy_session_id,
context.lease_id,
context.preparation,
context.now_ms,
reason,
)
}
};
let outcome = context.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
session: context.session,
lease_id: context.lease_id,
proxy_session_id: context.proxy_session_id,
preparation: context.preparation,
now_ms: context.ctx.hls_proxy.terminal_commit_now_ms().max(context.now_ms),
payload: HlsTerminalCommitPayload::CustomResponse(plan),
asset_revision_guard: terminal_asset_revision_guard(context.ctx, expected_asset.reason, Some(expected_asset)),
});
terminal_resolution_for_commit_outcome(outcome, context.now_ms)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum HlsAutonomousTerminalObservation {
Committed,
@@ -486,6 +527,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure(
lease_id: &HlsAccessLeaseId,
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
reason: HlsTerminalTailCompatibility,
) -> HlsTerminalResolution {
let expected_asset = configured_terminal_asset_identity(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable);
let outcome = ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
@@ -494,9 +536,7 @@ pub(super) fn commit_prepared_terminal_unavailable_after_owner_failure(
proxy_session_id,
preparation,
now_ms: ctx.hls_proxy.terminal_commit_now_ms().max(now_ms),
payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(
HlsTerminalTailCompatibility::TerminalMediaNotReady,
),
payload: HlsTerminalCommitPayload::UnavailableAfterOwnerFailure(reason),
asset_revision_guard: terminal_asset_revision_guard(
ctx,
HlsRuntimeCustomTailReason::ChannelUnavailable,
+1
View File
@@ -212,6 +212,7 @@ mod policy;
mod publication;
mod late_custom_response;
mod recovery;
mod storage;
@@ -0,0 +1,532 @@
use super::*;
use crate::{
late_custom_response::prepare_late_custom_response,
manager::{HlsTerminalCommitPayload, HlsTerminalCommitRequest},
};
async fn late_fixture(name: &str) -> HlsTerminalPendingCommitFixture {
let fixture = terminal_pending_commit_fixture(name).await;
fixture.session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict;
fixture
}
async fn prepare(
fixture: &HlsTerminalPendingCommitFixture,
) -> (Arc<crate::late_custom_response::HlsLateCustomResponsePlan>, HlsRuntimeCustomTailAssetIdentity) {
prepare_late_custom_response(
&fixture.ctx,
&fixture.proxy_session_id,
&fixture.lease_id,
&fixture.preparation,
fixture.now_ms,
)
.await
.unwrap_or_else(|_| panic!("late custom plan"))
}
fn commit(
fixture: &HlsTerminalPendingCommitFixture,
plan: Arc<crate::late_custom_response::HlsLateCustomResponsePlan>,
expected_asset: HlsRuntimeCustomTailAssetIdentity,
now_ms: u64,
) -> HlsTerminalCommitOutcome {
fixture.ctx.hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest {
session: &fixture.session,
lease_id: &fixture.lease_id,
proxy_session_id: &fixture.proxy_session_id,
preparation: &fixture.preparation,
now_ms,
payload: HlsTerminalCommitPayload::CustomResponse(plan),
asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, expected_asset.reason, Some(expected_asset)),
})
}
#[tokio::test]
async fn late_custom_response_commits_without_cached_base_and_serves_frozen_media() {
let fixture = late_fixture("late-without-base").await;
let (plan, asset) = prepare(&fixture).await;
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(asset.media);
assert!(
fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms)
.await
.is_err(),
"preparation cannot authorize uncommitted custom media"
);
fixture.session.write().await.segments.clear();
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::Committed);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:41\n"));
assert!(!plan.manifest_body().contains("/hls/shared/live/"));
let segment = fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms)
.await
.expect("committed late media");
assert!(!segment.bytes.is_empty());
fixture.ctx.app_config.custom_stream_response.store(None);
assert_eq!(commit(&fixture, Arc::clone(&plan), asset, now_ms + 1), HlsTerminalCommitOutcome::AlreadyCommitted);
let replay = fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 11, now_ms + 1)
.await
.expect("frozen media after config reload");
assert_eq!(replay.bytes, segment.bytes);
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, "wrong", 11, now_ms)
.await
.is_err());
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 12, now_ms)
.await
.is_err());
assert!(fixture
.ctx
.hls_proxy
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, fixture.now_ms + 60_000)
.await
.is_err());
}
#[tokio::test]
async fn late_custom_response_does_not_commit_over_recovered_origin_or_changed_asset() {
for changed_asset in [false, true] {
let fixture = late_fixture(if changed_asset { "late-asset-changed" } else { "late-origin-recovered" }).await;
let (plan, asset) = prepare(&fixture).await;
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let expected = if changed_asset {
fixture.ctx.app_config.custom_stream_response.store(None);
HlsTerminalCommitOutcome::SupersededGeneration
} else {
fixture.session.write().await.origin_control.record_media_progress(now_ms, 12_000);
HlsTerminalCommitOutcome::RecoveryCommitted
};
assert_eq!(commit(&fixture, plan, asset, now_ms), expected);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("lease");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
}
}
#[tokio::test]
async fn late_custom_response_preserves_the_prepared_asset_reason_guard() {
let fixture = late_fixture("late-wrong-asset-reason").await;
let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted)
.expect("configured alternate custom asset");
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(
commit_prepared_runtime_custom_tail(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
&fixture.preparation,
now_ms,
asset,
)
.await,
HlsTerminalResolution::Reevaluate
);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("live lease");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
}
#[tokio::test]
async fn late_custom_response_preserves_target_duration_exceeded_through_terminal_commit() {
let mut fixture = late_fixture("late-target-duration-error").await;
let mut manifest = fixture.preparation.manifest_snapshot.clone();
manifest.target_duration_ms = 1_000;
manifest.playlist_duration_ms = 1_000;
manifest.last_visible_media_end_ms = 1_000;
Arc::make_mut(&mut manifest.visible_segments)[0].duration_ms = 1_000;
let guard = fixture
.ctx
.hls_proxy
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("publication guard");
assert!(fixture
.ctx
.hls_proxy
.commit_access_lease_manifest_publication(
&fixture.lease_id,
&fixture.proxy_session_id,
guard,
manifest,
fixture.now_ms,
)
.await
.is_committed());
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("lease with short target duration");
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("published snapshot");
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
let expected = HlsTerminalTailCompatibility::TargetDurationExceeded {
asset_ms: fixture.asset.duration_ms(),
target_ms: 1_000,
};
assert!(matches!(prepare_late_custom_response(
&fixture.ctx, &fixture.proxy_session_id, &fixture.lease_id, &fixture.preparation, fixture.now_ms,
).await, Err(crate::late_custom_response::HlsLateCustomResponseError::Unavailable(reason)) if reason == expected));
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert_eq!(
super::super::terminal_pending_owner::commit_late_custom_response(HlsTerminalCommitContext {
ctx: &fixture.ctx,
session: &fixture.session,
proxy_session_id: &fixture.proxy_session_id,
lease_id: &fixture.lease_id,
preparation: &fixture.preparation,
now_ms,
})
.await,
HlsTerminalResolution::Committed
);
let terminal = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("terminal unavailable lease");
assert_eq!(
terminal.playback_mode,
HlsLeasePlaybackMode::TerminalUnavailable {
decision_generation: fixture.preparation.decision_generation,
reason: expected,
}
);
}
async fn committed_late_activity_fixture(name: &str) -> HlsTerminalPendingCommitFixture {
let fixture = late_fixture(name).await;
let mut config = fixture.ctx.app_config.config.load_full().as_ref().clone();
config.reverse_proxy =
Some(tuliprox_core::model::ReverseProxyConfig::from(&shared::model::ReverseProxyConfigDto {
hls_cache: Some(shared::model::HlsCacheConfigDto {
session_idle_timeout: shared::model::Secs::new(60),
..Default::default()
}),
..Default::default()
}));
fixture.ctx.app_config.config.store(Arc::new(config));
fixture.ctx.hls_proxy.update_config(&fixture.ctx.app_config).await;
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
assert!(fixture
.ctx
.hls_proxy
.activate_access_lease(
&fixture.lease_id,
&fixture.proxy_session_id,
now_ms,
HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 },
)
.await
.is_activated());
let (plan, asset) = prepare(&fixture).await;
assert_eq!(commit(&fixture, plan, asset, now_ms), HlsTerminalCommitOutcome::Committed);
fixture
}
#[tokio::test]
async fn late_custom_media_requests_keep_lease_and_session_alive_beyond_initial_sixty_seconds() {
use crate::{
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
HlsMediaActivityCommitOutcome,
};
let fixture = committed_late_activity_fixture("late-active-media").await;
let manager = &fixture.ctx.hls_proxy;
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let before = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms)
.await
.expect("terminal lease");
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
for index in 1_u16..=11 {
let now_ms = started_at_ms + u64::from(index) * 10_000;
let access = manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, index, now_ms)
.await
.expect("custom media remains available");
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, now_ms).await,
HlsMediaActivityCommitOutcome::Committed
);
if index == 6 {
for key in [
HlsLifecycleEventKey::AccessLeaseValidity {
lease_id: fixture.lease_id.clone(),
proxy_session_id: fixture.proxy_session_id.clone(),
},
HlsLifecycleEventKey::SessionIdle { proxy_session_id: fixture.proxy_session_id.clone() },
] {
manager
.handle_lifecycle_event(
&fixture.ctx.active_users,
&fixture.ctx.active_provider,
HlsLifecycleEvent { key, due_at_ms: started_at_ms + 60_000 },
now_ms,
)
.await;
}
}
let lease = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms)
.await
.expect("active terminal lease");
assert_eq!(lease.state, HlsAccessLeaseState::Activated);
assert_eq!(lease.last_seen_at_ms, now_ms);
assert_eq!(lease.valid_until_ms, now_ms + 60_000);
assert!(lease.active_until_ms.is_some_and(|until| until > now_ms));
assert_eq!(lease.playback_mode, before.playback_mode);
assert_eq!(lease.playback_cursor, before.playback_cursor);
assert_eq!(lease.admission_generation, before.admission_generation);
assert_eq!(fixture.session.read().await.last_client_access_at_ms, now_ms);
}
let expired_at_ms = started_at_ms + 170_000;
assert!(manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, expired_at_ms)
.await
.is_err());
let expired = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, expired_at_ms)
.await
.expect("expired lease");
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
assert_eq!(expired.last_seen_at_ms, started_at_ms + 110_000);
}
#[tokio::test]
async fn late_custom_media_activity_reactivates_idle_lease_but_rejects_stale_or_expired_access() {
use crate::{
lifecycle::{HlsLifecycleEvent, HlsLifecycleEventKey},
HlsMediaActivityCommitOutcome,
};
let fixture = committed_late_activity_fixture("late-idle-media").await;
let manager = &fixture.ctx.hls_proxy;
let started_at_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
manager
.handle_lifecycle_event(
&fixture.ctx.active_users,
&fixture.ctx.active_provider,
HlsLifecycleEvent {
key: HlsLifecycleEventKey::AccessLeaseActive {
lease_id: fixture.lease_id.clone(),
proxy_session_id: fixture.proxy_session_id.clone(),
},
due_at_ms: started_at_ms + 30_000,
},
started_at_ms + 40_000,
)
.await;
let idle = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("idle lease");
assert_eq!(idle.state, HlsAccessLeaseState::Idle);
let fingerprint = crate::runtime_custom_tail::standalone_custom_asset_fingerprint(fixture.expected_asset.media);
let mut access = manager
.resolve_standalone_custom_segment(&fixture.lease_id, &fingerprint, 0, started_at_ms + 40_000)
.await
.expect("idle custom media");
let original_generation = access.terminal_generation;
access.terminal_generation = Some(crate::HlsTerminalTailGeneration(fixture.preparation.decision_generation + 1));
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
);
let unchanged = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("unchanged idle lease");
assert_eq!(unchanged.last_seen_at_ms, idle.last_seen_at_ms);
assert_eq!(unchanged.valid_until_ms, idle.valid_until_ms);
access.terminal_generation = original_generation;
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 40_000).await,
HlsMediaActivityCommitOutcome::Committed
);
let active = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 40_000)
.await
.expect("reactivated lease");
assert_eq!(active.state, HlsAccessLeaseState::Activated);
assert_eq!(active.valid_until_ms, started_at_ms + 100_000);
assert_eq!(
manager.record_standalone_custom_segment_activity(&fixture.lease_id, &access, started_at_ms + 100_000).await,
HlsMediaActivityCommitOutcome::StaleLeaseIdentity
);
let expired = manager
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, started_at_ms + 100_000)
.await
.expect("expired lease");
assert_eq!(expired.state, HlsAccessLeaseState::Expired);
assert_eq!(expired.last_seen_at_ms, active.last_seen_at_ms);
assert_eq!(expired.valid_until_ms, active.valid_until_ms);
}
#[tokio::test]
async fn late_custom_response_retries_contended_commit_once_and_remains_immutable() {
let mut fixture = late_fixture("late-contended-commit").await;
let now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms();
fixture.preparation.cutover_timing =
HlsLeaseCutoverTiming::from_reserve(now_ms, 0, fixture.preparation.reserve.transition_margin, None);
let (plan, asset) = prepare(&fixture).await;
let guard = fixture.session.read().await;
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
assert!(matches!(commit(&fixture, Arc::clone(&plan), asset, now_ms), HlsTerminalCommitOutcome::LockBusy { .. }));
drop(guard);
let committed = tokio::time::timeout(Duration::from_secs(3), async {
loop {
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(
&fixture.lease_id,
&fixture.proxy_session_id,
fixture.ctx.hls_proxy.terminal_commit_now_ms(),
)
.await
.expect("lease");
if let HlsLeasePlaybackMode::TerminalCustomResponse(committed) = lease.playback_mode {
break committed;
}
tokio::task::yield_now().await;
}
})
.await
.expect("autonomous late custom commit");
assert_eq!(committed, plan);
assert_eq!(
commit(&fixture, plan, asset, fixture.ctx.hls_proxy.terminal_commit_now_ms()),
HlsTerminalCommitOutcome::AlreadyCommitted
);
}
#[tokio::test]
async fn late_custom_response_uses_highwater_even_when_latest_manifest_is_shorter() {
let mut fixture = late_fixture("late-highwater").await;
for (last_seq, discontinuity_sequence, discontinuity_before) in [(100, 7, true), (50, 3, false)] {
let guard = fixture
.ctx
.hls_proxy
.prepare_access_lease_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("publication guard");
let mut manifest = fixture.preparation.manifest_snapshot.clone();
manifest.first_proxy_seq = last_seq;
manifest.last_proxy_seq = last_seq;
manifest.discontinuity_sequence = discontinuity_sequence;
let segment = Arc::make_mut(&mut manifest.visible_segments).first_mut().expect("segment");
segment.proxy_seq = last_seq;
segment.discontinuity_before = discontinuity_before;
assert!(fixture
.ctx
.hls_proxy
.commit_access_lease_manifest_publication(
&fixture.lease_id,
&fixture.proxy_session_id,
guard,
manifest,
fixture.now_ms
)
.await
.is_committed());
}
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("lease");
fixture.preparation.manifest_snapshot = lease.last_manifest_snapshot.expect("latest snapshot");
fixture.preparation.manifest_snapshot_generation = fixture.preparation.manifest_snapshot.snapshot_generation;
let (plan, _) = prepare(&fixture).await;
assert!(plan.manifest_body().contains("#EXT-X-MEDIA-SEQUENCE:101\n"));
assert!(plan.manifest_body().contains("#EXT-X-DISCONTINUITY-SEQUENCE:8\n"));
}
#[tokio::test]
async fn late_custom_response_cas_budget_cannot_be_renewed_or_poisoned_by_polls() {
use crate::terminal_commit::{
terminal_commit_retry_schedule_budget_ms, HlsTerminalCommitCommand, HlsTerminalCommitOwnerKey,
HlsTerminalCommitRetryCoordinator, HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision,
};
let fixture = late_fixture("late-bounded-cas").await;
let (plan, asset) = prepare(&fixture).await;
let coordinator = HlsTerminalCommitRetryCoordinator::default();
let now_ms = fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch();
let command = |decision| {
let (cancellation_epoch, submission_token) = coordinator.reserve_submission().expect("submission binding");
HlsTerminalCommitCommand {
key: HlsTerminalCommitOwnerKey {
proxy_session_id: fixture.proxy_session_id.clone(),
lease_id: fixture.lease_id.clone(),
lease_issued_at_ms: fixture.preparation.lease_issued_at_ms,
decision_generation: fixture.preparation.decision_generation,
},
session: Arc::clone(&fixture.session),
session_incarnation: fixture
.ctx
.hls_proxy
.sessions()
.session_incarnation(&fixture.session)
.expect("session incarnation"),
preparation: fixture.preparation.clone(),
decision,
media_guard: None,
asset_revision_guard: terminal_asset_revision_guard(&fixture.ctx, asset.reason, Some(asset)),
cancellation_epoch,
submission_token,
}
};
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
now_ms
),
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
));
assert!(matches!(
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms),
HlsTerminalCommitSubmissionDecision::Attempt { .. }
));
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady)),
now_ms + 1
),
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
));
assert!(matches!(
coordinator.submit(command(HlsTerminalLeaseDecision::CustomResponse(Arc::clone(&plan))), now_ms + 100),
HlsTerminalCommitSubmissionDecision::PendingExisting { .. }
));
assert!(matches!(
coordinator.submit(
command(HlsTerminalLeaseDecision::CustomResponse(plan)),
now_ms + terminal_commit_retry_schedule_budget_ms()
),
HlsTerminalCommitSubmissionDecision::Failed(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed)
));
}
@@ -31,6 +31,7 @@ pub(super) fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) -
manifest_commit_requirement:
super::super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed,
fresh_manifest_requirement_generation: None,
refresh_identity: None,
acceptance_directive: HlsManifestAcceptanceDirective::none(),
access_lease_id: None,
now_ms: fixture.now_ms,
@@ -386,3 +386,287 @@ fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_e
assert!(pressure.decision.close_admission);
assert!(!pressure.decision.evaluate_lease_cutovers);
}
async fn wait_for_terminal_mode(fixture: &PostRefreshTerminalFixture) -> HlsLeasePlaybackMode {
tokio::time::timeout(Duration::from_millis(4_500), async {
loop {
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("active lease");
if lease.playback_mode != HlsLeasePlaybackMode::Live {
return lease.playback_mode;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("terminal decision before reserve is exhausted")
}
#[tokio::test]
async fn live_reserve_deadline_commits_without_another_client_poll_while_refresh_is_in_flight() {
let fixture = post_refresh_terminal_fixture_with_progress("autonomous-reserve-deadline", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
fixture.now_ms,
)
.await;
{
let mut session = fixture.session.write().await;
session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure;
session.origin_refresh.mark_started(fixture.now_ms);
}
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
let result = commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease,
fixture.now_ms,
)
.await;
assert_eq!(result, HlsTerminalResolution::LiveAllowed);
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
// R=17,100 ms and M=12,000 ms: the next six-second client poll would be too late.
// Only read the resulting state; do not issue another terminal/manifest request.
let mode = wait_for_terminal_mode(&fixture).await;
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
assert!(current_time_millis() < fixture.now_ms + 5_100);
assert!(fixture.session.read().await.origin_refresh.in_flight);
}
#[tokio::test]
async fn first_http_502_arms_cutover_before_retry_after_sleep_finishes() {
use std::sync::atomic::{AtomicUsize, Ordering};
let fixture = post_refresh_terminal_fixture_with_progress("first-http-failure-cutover", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
fixture.now_ms,
)
.await;
fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms, 12_000);
let hits = Arc::new(AtomicUsize::new(0));
let handler_hits = Arc::clone(&hits);
let router = axum::Router::new().route(
"/index.m3u8",
axum::routing::get(move || {
let handler_hits = Arc::clone(&handler_hits);
async move {
let status = if handler_hits.fetch_add(1, Ordering::SeqCst) == 0 {
axum::http::StatusCode::BAD_GATEWAY
} else {
axum::http::StatusCode::NOT_FOUND
};
(status, [(axum::http::header::RETRY_AFTER, "60")], "origin failure")
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("local origin binds");
let address = listener.local_addr().expect("local origin address");
let server = tokio::spawn(async move {
axum::serve(listener, router).await.expect("local origin serves");
});
let mut request = post_refresh_owner_request(&fixture);
request.origin_entry = crate::manifest_fetch::LiveHlsOriginEntry::parse(&format!("http://{address}/index.m3u8"))
.expect("local origin URL");
let refresh = tokio::spawn(crate::refresh::trigger_origin_refresh_sync(request));
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if fixture.session.read().await.origin_control.path_condition
== HlsOriginPathCondition::RetryableFetchFailure
&& fixture.ctx.hls_proxy.availability_reevaluations().owner_count() == 1
{
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("first failed attempt publishes degraded evidence and owns the deadline");
assert_eq!(hits.load(Ordering::SeqCst), 1);
assert!(fixture.session.read().await.origin_refresh.in_flight);
let mode = wait_for_terminal_mode(&fixture).await;
assert!(matches!(mode, HlsLeasePlaybackMode::TerminalTail(_)), "{mode:?}");
assert!(current_time_millis() < fixture.now_ms + 5_100);
assert_eq!(hits.load(Ordering::SeqCst), 1, "cutover must precede the capped six-second retry");
assert!(fixture.session.read().await.origin_refresh.in_flight);
assert!(tokio::time::timeout(Duration::from_secs(8), refresh)
.await
.expect("refresh finishes")
.expect("refresh task"));
server.abort();
}
#[tokio::test]
async fn recovered_origin_cancels_the_retained_reserve_cutover() {
let fixture = post_refresh_terminal_fixture_with_progress("recovered-reserve-owner", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
18_000,
fixture.now_ms,
)
.await;
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease,
fixture.now_ms,
)
.await,
HlsTerminalResolution::LiveAllowed
);
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
fixture.session.write().await.origin_control.record_media_progress(current_time_millis(), 12_000);
fixture.ctx.hls_proxy.availability_reevaluations().notify_session_evidence_changed(&fixture.proxy_session_id);
wait_for_availability_owner_completion(&fixture).await;
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("recovered lease");
assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live);
}
#[tokio::test]
async fn expired_splice_deadline_finishes_custom_response_once_instead_of_leaving_a_live_lease() {
let fixture = post_refresh_terminal_fixture_with_progress("expired-splice-unavailable", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
26_000,
fixture.now_ms,
)
.await;
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease,
fixture.now_ms,
)
.await,
HlsTerminalResolution::Committed
);
let terminal = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("terminal lease");
assert!(matches!(terminal.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&terminal,
current_time_millis(),
)
.await,
HlsTerminalResolution::Committed
);
assert!(!fixture.session.read().await.has_terminal_tail_protections());
}
#[tokio::test(start_paused = true)]
async fn post_refresh_recovery_request_replaces_a_reserve_only_worker_without_losing_refresh_work() {
let fixture = post_refresh_terminal_fixture_with_progress("reserve-owner-recovery-upgrade", true, false).await;
fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease,
fixture.now_ms,
)
.await,
HlsTerminalResolution::LiveAllowed
);
assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1);
assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await;
}
#[tokio::test]
async fn publication_late_origin_can_finish_custom_response_after_the_splice_deadline() {
let fixture = post_refresh_terminal_fixture_with_progress("publication-late-unavailable", true, false).await;
advance_post_refresh_fixture_playback(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&fixture.lease_id,
26_000,
fixture.now_ms,
)
.await;
fixture.session.write().await.origin_control.record_media_progress(fixture.now_ms - 18_000, 12_000);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms)
.await
.expect("live lease");
assert_eq!(
commit_terminal_tail_if_lease_reserve_requires_cutover(
&fixture.ctx,
&fixture.session,
&fixture.proxy_session_id,
&lease,
fixture.now_ms,
)
.await,
HlsTerminalResolution::Committed
);
let lease = fixture
.ctx
.hls_proxy
.access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, current_time_millis())
.await
.expect("terminal lease");
assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalCustomResponse(_)));
}
@@ -63,6 +63,7 @@ pub(super) async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFi
let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode {
HlsLeasePlaybackMode::Live => "live",
HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail",
HlsLeasePlaybackMode::TerminalCustomResponse(_) => "terminal-custom-response",
HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable",
HlsLeasePlaybackMode::Ended => "ended",
});
@@ -332,7 +333,7 @@ pub(super) async fn assert_multi_lease_fallback_handles_pending_and_unavailable(
assert_eq!(aggregate.total, 2);
assert_eq!(aggregate.pending_owned, 1);
assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}");
assert!(aggregate.unresolved.is_empty());
assert_eq!(aggregate.unresolved.len(), 0);
assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1);
let unavailable = fixture
.ctx
@@ -148,6 +148,7 @@ pub(super) async fn install_terminal_pending_ready_base(
host_local_index: 0,
},
proxy_seq: 40,
media_start_ms: 0,
duration_ms: asset.duration_ms(),
proxy_file_ext: "ts".to_string(),
content_type: "video/mp2t".to_string(),
@@ -316,7 +317,8 @@ pub(super) async fn assert_terminal_pending_registration_failure_commits_unavail
fixture.expected_asset,
fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(),
failure,
);
)
.await;
assert_eq!(resolution, HlsTerminalResolution::Committed);
let lease = fixture
@@ -340,6 +342,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
let mut session = fixture.session.write().await;
session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict;
let origin_epoch = session.origin_control.origin_epoch;
let base_end_ms = session.segments[&40].media_start_ms + session.segments[&40].duration_ms;
for proxy_seq in 41_u64..=44 {
session.segments.insert(
proxy_seq,
@@ -351,6 +354,7 @@ pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_l
host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX),
},
proxy_seq,
media_start_ms: base_end_ms + (proxy_seq - 41) * 20_000,
duration_ms: 20_000,
proxy_file_ext: "ts".to_string(),
content_type: "video/mp2t".to_string(),
+26 -17
View File
@@ -81,6 +81,8 @@ pub enum HlsAvailabilityReevaluationRegistration {
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum HlsAvailabilityReevaluationMode {
RecoveryPressure,
/// Tracks a live reserve deadline independently of a client request or a running origin fetch.
TerminalReserve,
PostRefresh(HlsPostRefreshAvailabilityReason),
}
@@ -90,8 +92,10 @@ impl HlsAvailabilityReevaluationMode {
(Self::PostRefresh(current), Self::PostRefresh(incoming)) => {
Self::PostRefresh(stronger_post_refresh_reason(current, incoming))
}
(Self::PostRefresh(reason), Self::RecoveryPressure)
| (Self::RecoveryPressure, Self::PostRefresh(reason)) => Self::PostRefresh(reason),
(Self::PostRefresh(reason), Self::RecoveryPressure | Self::TerminalReserve)
| (Self::RecoveryPressure | Self::TerminalReserve, Self::PostRefresh(reason)) => Self::PostRefresh(reason),
(Self::TerminalReserve, Self::RecoveryPressure | Self::TerminalReserve)
| (Self::RecoveryPressure, Self::TerminalReserve) => Self::TerminalReserve,
(Self::RecoveryPressure, Self::RecoveryPressure) => Self::RecoveryPressure,
}
}
@@ -106,10 +110,14 @@ const fn stronger_post_refresh_reason(
| (_, HlsPostRefreshAvailabilityReason::HardManifestFailure) => {
HlsPostRefreshAvailabilityReason::HardManifestFailure
}
(HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, _)
| (_, HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict) => {
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict
}
(
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
) => HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict,
HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
) => HlsPostRefreshAvailabilityReason::RetryableManifestFailure,
}
}
@@ -174,7 +182,7 @@ impl HlsAvailabilityReevaluationCoordinator {
pub fn register<Work, WorkFuture>(
self: &Arc<Self>,
key: HlsAvailabilityReevaluationOwnerKey,
mode: HlsAvailabilityReevaluationMode,
mut mode: HlsAvailabilityReevaluationMode,
work: Work,
) -> HlsAvailabilityReevaluationRegistration
where
@@ -186,24 +194,25 @@ impl HlsAvailabilityReevaluationCoordinator {
};
let mut state = self.lock_state();
if let Some(current) = state.owners.get_mut(&key.proxy_session_id) {
if current.key == key {
current.mode = current.mode.merge(mode);
current.rerun_requested = true;
current.wake.notify_one();
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
}
if !key.supersedes(&current.key) {
if current.key != key && !key.supersedes(&current.key) {
return HlsAvailabilityReevaluationRegistration::Superseded;
}
let installs_refresh_payload = current.mode == HlsAvailabilityReevaluationMode::TerminalReserve
&& mode != HlsAvailabilityReevaluationMode::TerminalReserve;
mode = current.mode.merge(mode);
// A newer demand for the same proxy session is owned by the
// already-counted task. The worker will snapshot current session
// evidence and atomically rekey itself without acquiring another
// permit. Keeping one dirty bit preserves a bounded successor
// cycle while coalescing repeated registrations.
current.mode = current.mode.merge(mode);
current.rerun_requested = true;
current.wake.notify_one();
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
// A reserve-only worker has no origin request payload. Replace it
// when recovery work arrives so merging the mode cannot discard that work.
if !installs_refresh_payload {
current.mode = mode;
current.rerun_requested = true;
current.wake.notify_one();
return HlsAvailabilityReevaluationRegistration::AlreadyOwned;
}
}
let Ok(task_permit) = Arc::clone(&self.task_capacity).try_acquire_owned() else {
return HlsAvailabilityReevaluationRegistration::CapacityExceeded;
+30 -2
View File
@@ -48,6 +48,8 @@ pub struct StagedCacheObject {
pub size: u64,
cache_path_generation: Arc<CachePathGeneration>,
_registration: Arc<ActiveTempFileRegistration>,
pub(crate) write_budget: Option<Arc<HlsRevisionDiskReservation>>,
cleanup: Arc<StagedFileCleanup>,
}
/// Typed source used to distinguish a decoded-object size violation from a filesystem failure.
@@ -106,6 +108,10 @@ pub struct HlsSegmentCache {
capacity_reclaimer: StdRwLock<Option<Weak<dyn HlsCacheCapacityReclaimer>>>,
capacity_admission_gate: AsyncMutex<()>,
owned_operation_permits: Arc<Semaphore>,
#[cfg(test)]
startup_test_checkpoint: StdMutex<Option<test_support::StartupWriteCheckpoint>>,
#[cfg(test)]
delete_test_failures: StdMutex<HashMap<PathBuf, usize>>,
}
#[derive(Clone)]
@@ -125,8 +131,13 @@ struct CacheCapacityState {
revision: Arc<CapacityRevision>,
active_mutations: HashSet<PathBuf>,
revision_reservations: HashMap<u64, (String, u64)>,
/// Revision file bytes found by the last completed usage scan.
next_write_id: u64,
write_owners: HashMap<PathBuf, u64>,
abandoned_writes: HashMap<PathBuf, (u64, String)>,
reserved_spools: HashMap<PathBuf, u64>,
/// Current revision bytes, reconciled by scans and successful file mutations.
revision_bytes: u64,
revision_metrics: Option<Weak<crate::HlsCacheMetrics>>,
}
#[derive(Default)]
@@ -135,17 +146,24 @@ struct CapacityRevision;
#[derive(Default)]
struct TempFileState {
active_files: HashSet<PathBuf>,
revision: Arc<CapacityRevision>,
deletion_reservations: HashSet<PathBuf>,
}
struct ActiveTempFileRegistration {
path: PathBuf,
state: Arc<StdMutex<TempFileState>>,
changed: Arc<Notify>,
}
impl Drop for ActiveTempFileRegistration {
fn drop(&mut self) {
self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.remove(&self.path);
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if state.active_files.remove(&self.path) {
state.revision = Arc::new(CapacityRevision);
drop(state);
self.changed.notify_waiters();
}
}
}
@@ -171,6 +189,8 @@ struct CapacityMutationReservation {
session_component: String,
replacement: Option<(u64, u64)>,
filesystem_mutation_started: bool,
write_budget: Option<Arc<HlsRevisionDiskReservation>>,
transferred_bytes: u64,
capacity: Arc<StdMutex<CacheCapacityState>>,
changed: Arc<Notify>,
}
@@ -198,6 +218,8 @@ pub struct HlsRevisionDiskReservation {
changed: Arc<Notify>,
/// Set when bytes were written under this reservation outside the staged commit accounting.
unaccounted_writes: bool,
generation: Arc<CachePathGeneration>,
cache_path: PathBuf,
}
pub struct HlsRevisionFilePin {
@@ -217,6 +239,9 @@ mod lifecycle;
mod revisions;
mod staging;
#[cfg(test)]
mod test_support;
mod write_budget;
#[cfg(test)]
use capacity::CapacityReservationError;
#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")]
pub use capacity::{
@@ -226,3 +251,6 @@ pub use capacity::{
pub use error::{hls_cache_capacity_from_io, hls_cache_object_limit_from_io};
pub use keys::HlsCacheObjectKey;
pub use staging::CachedSegmentMetadata;
#[cfg(test)]
pub(crate) use test_support::StartupWritePhase;
use write_budget::StagedFileCleanup;
+168 -22
View File
@@ -1,7 +1,7 @@
use super::{
error::{cache_object_limit_error, capacity_error},
hls_cache_capacity_from_io,
lifecycle::{is_temp_cache_file, REWRITE_SECRET_FINGERPRINT_FILE},
lifecycle::REWRITE_SECRET_FINGERPRINT_FILE,
revisions::{is_revision_cache_path, revision_reserved_bytes},
safe_proxy_session_id, CacheCapacityPressure, CacheCapacityState, CapacityInvalidationGuard,
CapacityMutationReservation, CapacityRevision, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsSegmentCache,
@@ -82,11 +82,21 @@ impl CapacityMutationReservation {
return Err(CapacityReservationError::Invalidated);
}
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size);
let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size);
// Old bytes remain on disk until rename succeeds. Reserve growth now,
// but publish any shrink only after the filesystem mutation completes.
let growth = new_size.saturating_sub(old_size);
let projected_total = capacity.total_bytes.saturating_add(growth);
let projected_session = session_size.saturating_add(growth);
let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &self.session_component);
if projected_total.saturating_add(reserved_global) > max_cache_bytes
|| projected_session.saturating_add(reserved_session) > max_session_bytes
let transferred = self
.write_budget
.as_ref()
.and_then(|budget| {
capacity.revision_reservations.get(&budget.id).filter(|(owner, _)| owner == &self.session_component)
})
.map_or(0, |(_, bytes)| (*bytes).min(new_size.saturating_sub(old_size)));
if projected_total.saturating_add(reserved_global.saturating_sub(transferred)) > max_cache_bytes
|| projected_session.saturating_add(reserved_session.saturating_sub(transferred)) > max_session_bytes
{
return Err(CapacityReservationError::Exceeded {
pressure: CacheCapacityPressure {
@@ -96,10 +106,10 @@ impl CapacityMutationReservation {
current_global_bytes: capacity.total_bytes,
staged_bytes: new_size,
required_session_bytes: projected_session
.saturating_add(reserved_session)
.saturating_add(reserved_session.saturating_sub(transferred))
.saturating_sub(max_session_bytes),
required_global_bytes: projected_total
.saturating_add(reserved_global)
.saturating_add(reserved_global.saturating_sub(transferred))
.saturating_sub(max_cache_bytes),
},
// Capture the opaque token in the same critical section as the
@@ -108,6 +118,12 @@ impl CapacityMutationReservation {
revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
});
}
if let Some(budget) = &self.write_budget {
if let Some((_, bytes)) = capacity.revision_reservations.get_mut(&budget.id) {
*bytes -= transferred;
}
}
self.transferred_bytes = transferred;
capacity.total_bytes = projected_total;
store_session_bytes(&mut capacity, &self.session_component, projected_session);
self.replacement = Some((old_size, new_size));
@@ -117,12 +133,36 @@ impl CapacityMutationReservation {
pub(super) fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; }
pub(super) fn finish_replacement(mut self) {
let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((old, new)) = self.replacement {
let released = old.saturating_sub(new);
capacity.total_bytes = capacity.total_bytes.saturating_sub(released);
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(released));
if super::revisions::is_revision_cache_path(&self.path) {
capacity.revision_bytes = capacity.revision_bytes.saturating_sub(old).saturating_add(new);
}
}
capacity.abandoned_writes.remove(&self.path);
if let Some(budget) = &self.write_budget {
capacity.write_owners.insert(self.path.clone(), budget.id);
} else {
capacity.write_owners.remove(&self.path);
}
drop(capacity);
self.replacement = None;
self.finish_mutation(None);
}
pub(super) fn finish_delete(mut self, deleted_size: u64) {
self.finish_mutation(Some(deleted_size));
let unaccounted = self
.capacity
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.reserved_spools
.remove(&self.path)
.is_some();
self.finish_mutation(Some(if unaccounted { 0 } else { deleted_size }));
self.replacement = None;
}
@@ -134,11 +174,17 @@ impl CapacityMutationReservation {
capacity.active_mutations.remove(&self.path);
if capacity.initialized && capacity.cache_path == self.cache_path {
if let Some(deleted_size) = deleted_size {
capacity.write_owners.remove(&self.path);
capacity.abandoned_writes.remove(&self.path);
if super::revisions::is_revision_cache_path(&self.path) {
capacity.revision_bytes = capacity.revision_bytes.saturating_sub(deleted_size);
}
capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size);
let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size));
}
}
update_revision_metrics(&capacity);
capacity.revision = Arc::new(CapacityRevision);
drop(capacity);
self.changed.notify_waiters();
@@ -152,18 +198,22 @@ impl Drop for CapacityMutationReservation {
return;
}
capacity.active_mutations.remove(&self.path);
if !self.filesystem_mutation_started && self.transferred_bytes > 0 {
if let Some((_, bytes)) =
self.write_budget.as_ref().and_then(|budget| capacity.revision_reservations.get_mut(&budget.id))
{
*bytes = bytes.saturating_add(self.transferred_bytes);
}
}
let revision_changed = if self.filesystem_mutation_started {
capacity.initialized = false;
true
} else if capacity.initialized && capacity.cache_path == self.cache_path {
if let Some((old_size, new_size)) = self.replacement.take() {
capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size);
let growth = new_size.saturating_sub(old_size);
capacity.total_bytes = capacity.total_bytes.saturating_sub(growth);
let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default();
store_session_bytes(
&mut capacity,
&self.session_component,
session_size.saturating_sub(new_size).saturating_add(old_size),
);
store_session_bytes(&mut capacity, &self.session_component, session_size.saturating_sub(growth));
true
} else {
false
@@ -342,13 +392,36 @@ impl HlsSegmentCache {
notified.await;
continue;
};
let (total_bytes, session_bytes, revision_bytes) = scan_committed_cache_usage(cache_path).await?;
let (temp_revision, mut excluded) = {
let temps = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let excluded: std::collections::HashSet<_> = temps
.active_files
.iter()
.filter(|path| super::lifecycle::is_temp_cache_file(path))
.cloned()
.collect();
(Arc::clone(&temps.revision), excluded)
};
excluded.extend(
self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reserved_spools.keys().cloned(),
);
#[cfg(test)]
self.startup_write_checkpoint(super::StartupWritePhase::AfterInventorySnapshot).await?;
let (total_bytes, session_bytes, revision_bytes) =
scan_cache_usage_excluding(cache_path, &excluded).await?;
if self.cache_path_snapshot().path != cache_path {
return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan"));
}
// Registration precedes file creation. Changes after this check cannot
// have affected the completed scan; no nested temp/capacity locks are needed.
let temps_changed = !Arc::ptr_eq(
&self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).revision,
&temp_revision,
);
let scan_invalidated = {
let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if !Arc::ptr_eq(&capacity.revision, &revision)
if temps_changed
|| !Arc::ptr_eq(&capacity.revision, &revision)
|| capacity.active_mutations.iter().any(|path| path.starts_with(cache_path))
{
true
@@ -358,6 +431,18 @@ impl HlsSegmentCache {
capacity.total_bytes = total_bytes;
capacity.session_bytes = session_bytes;
capacity.revision_bytes = revision_bytes;
let sessions: std::collections::HashSet<_> = capacity.session_bytes.keys().cloned().collect();
let active_owners: std::collections::HashSet<_> =
capacity.revision_reservations.keys().copied().collect();
capacity.write_owners.retain(|path, owner| {
active_owners.contains(owner)
|| path.strip_prefix(cache_path).ok().and_then(|path| path.components().next()).is_some_and(
|session| sessions.contains(session.as_os_str().to_string_lossy().as_ref()),
)
});
let owners = capacity.write_owners.clone();
capacity.abandoned_writes.retain(|path, (owner, _)| owners.get(path) == Some(owner));
update_revision_metrics(&capacity);
false
}
};
@@ -407,6 +492,8 @@ impl HlsSegmentCache {
session_component,
replacement: None,
filesystem_mutation_started: false,
write_budget: None,
transferred_bytes: 0,
capacity: Arc::clone(&self.capacity),
changed: Arc::clone(&self.capacity_changed),
})
@@ -421,9 +508,39 @@ impl HlsSegmentCache {
max_cache_bytes: u64,
max_session_bytes: u64,
) -> io::Result<CapacityMutationReservation> {
self.prepare_capacity_replacement_with_budget(
cache_path,
final_path,
session_component,
new_size,
max_cache_bytes,
max_session_bytes,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn prepare_capacity_replacement_with_budget(
&self,
cache_path: &Path,
final_path: &Path,
session_component: String,
new_size: u64,
max_cache_bytes: u64,
max_session_bytes: u64,
budget: Option<Arc<super::HlsRevisionDiskReservation>>,
) -> io::Result<CapacityMutationReservation> {
if budget.as_ref().is_some_and(|budget| {
!Arc::ptr_eq(&budget.capacity, &self.capacity)
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
}) {
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation"));
}
loop {
self.ensure_capacity_initialized(cache_path).await?;
let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await;
reservation.write_budget = budget.clone();
let old_size = match fs::metadata(final_path).await {
Ok(metadata) => metadata.len(),
Err(err) if err.kind() == io::ErrorKind::NotFound => 0,
@@ -465,7 +582,15 @@ fn invalidate_capacity_state(capacity: &StdMutex<CacheCapacityState>, changed: &
changed.notify_waiters();
}
#[cfg(test)]
pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap<String, u64>, u64)> {
scan_cache_usage_excluding(cache_path, &std::collections::HashSet::new()).await
}
async fn scan_cache_usage_excluding(
cache_path: &Path,
excluded: &std::collections::HashSet<PathBuf>,
) -> io::Result<(u64, HashMap<String, u64>, u64)> {
let mut total_bytes = 0_u64;
let mut revision_bytes = 0_u64;
let mut session_bytes = HashMap::new();
@@ -475,22 +600,37 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<
Err(err) => return Err(err),
};
while let Some(root_entry) = root_entries.next_entry().await? {
if !root_entry.file_type().await?.is_dir() {
continue;
match root_entry.file_type().await {
Ok(kind) if kind.is_dir() => {}
Ok(_) => continue,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
}
let session_component = root_entry.file_name().to_string_lossy().into_owned();
let mut bytes = 0_u64;
let mut pending_dirs = vec![root_entry.path()];
while let Some(dir) = pending_dirs.pop() {
let mut entries = fs::read_dir(dir).await?;
let mut entries = match fs::read_dir(dir).await {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
};
while let Some(entry) = entries.next_entry().await? {
let file_type = entry.file_type().await?;
let file_type = match entry.file_type().await {
Ok(kind) => kind,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
};
if file_type.is_dir() {
pending_dirs.push(entry.path());
} else if file_type.is_file() {
let revision = is_revision_cache_path(&entry.path());
if revision || !is_temp_cache_file(&entry.path()) {
let size = entry.metadata().await?.len();
if !excluded.contains(&entry.path()) {
let size = match entry.metadata().await {
Ok(metadata) => metadata.len(),
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
};
bytes = bytes.saturating_add(size);
if revision {
revision_bytes = revision_bytes.saturating_add(size);
@@ -506,3 +646,9 @@ pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<
}
Ok((total_bytes, session_bytes, revision_bytes))
}
pub(super) fn update_revision_metrics(capacity: &CacheCapacityState) {
if let Some(metrics) = capacity.revision_metrics.as_ref().and_then(Weak::upgrade) {
metrics.record_revision_disk_bytes(capacity.revision_bytes);
}
}
+30 -12
View File
@@ -1,9 +1,9 @@
use super::{
revisions::is_revision_cache_path, staging::run_owned_cache_operation, ActiveTempFileRegistration,
CacheCapacityState, CacheInvalidationOutcome, CachePathDeletionReservation, CachePathGeneration, CachePathState,
CachedSegmentMetadata, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer,
HlsCacheCapacityRevision, HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState,
DEFAULT_HLS_CACHE_PATH, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN,
staging::run_owned_cache_operation, ActiveTempFileRegistration, CacheCapacityState, CacheInvalidationOutcome,
CachePathDeletionReservation, CachePathGeneration, CachePathState, CachedSegmentMetadata,
CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, HlsCacheCapacityRevision,
HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, DEFAULT_HLS_CACHE_PATH,
MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN,
};
use log::warn;
use std::{
@@ -41,6 +41,10 @@ impl HlsSegmentCache {
capacity_reclaimer: StdRwLock::new(None),
capacity_admission_gate: AsyncMutex::new(()),
owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)),
#[cfg(test)]
startup_test_checkpoint: StdMutex::new(None),
#[cfg(test)]
delete_test_failures: StdMutex::new(std::collections::HashMap::new()),
}
}
@@ -123,6 +127,7 @@ impl HlsSegmentCache {
pub async fn delete<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<()> {
let cache_path = self.cache_path_snapshot();
self.ensure_capacity_initialized(&cache_path.path).await?;
let path = Self::path_for_key_in(&cache_path.path, key);
let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await;
self.delete_with_reservation(path, reservation).await
@@ -130,6 +135,7 @@ impl HlsSegmentCache {
pub async fn delete_if_inactive<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<()> {
let cache_path = self.cache_path_snapshot();
self.ensure_capacity_initialized(&cache_path.path).await?;
let path = Self::path_for_key_in(&cache_path.path, key);
let reservation = self
.try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component())
@@ -142,6 +148,8 @@ impl HlsSegmentCache {
path: PathBuf,
reservation: CapacityMutationReservation,
) -> io::Result<()> {
#[cfg(test)]
self.check_cache_delete_test_failure(&path)?;
run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move {
let size = match fs::metadata(&path).await {
Ok(metadata) => metadata.len(),
@@ -190,9 +198,7 @@ impl HlsSegmentCache {
let cache_path = self.cache_path_snapshot();
let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?;
let mut deleted = 0_usize;
let mut removed_revision = false;
for path in candidates {
let revision = is_revision_cache_path(&path);
let Some(deletion) = self.reserve_path_deletion(&path) else {
continue;
};
@@ -209,14 +215,14 @@ impl HlsSegmentCache {
match result {
Ok(()) => {
deleted = deleted.saturating_add(1);
removed_revision |= revision;
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()),
}
}
if removed_revision {
if deleted > 0 {
drop(self.capacity_invalidation_guard());
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?;
}
Ok(deleted)
}
@@ -238,7 +244,8 @@ impl HlsSegmentCache {
drop(deletion);
result
})
.await
.await?;
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await
}
pub async fn delete_orphan_session_dirs(
@@ -307,6 +314,9 @@ impl HlsSegmentCache {
}
}
}
if removed > 0 {
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await?;
}
Ok(removed)
}
@@ -389,7 +399,8 @@ impl HlsSegmentCache {
drop(deletion);
result
})
.await
.await?;
self.ensure_capacity_initialized(&self.cache_path_snapshot().path).await
}
pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result<Option<String>> {
@@ -422,7 +433,14 @@ impl HlsSegmentCache {
if !state.active_files.insert(path.to_path_buf()) {
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active"));
}
Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) }))
state.revision = Arc::new(CapacityRevision);
drop(state);
self.capacity_changed.notify_waiters();
Ok(Arc::new(ActiveTempFileRegistration {
path: path.to_path_buf(),
state: Arc::clone(&self.temp_files),
changed: Arc::clone(&self.capacity_changed),
}))
}
pub(super) fn reserve_path_deletion(&self, path: &Path) -> Option<CachePathDeletionReservation> {
@@ -1,5 +1,81 @@
use super::*;
#[test]
fn temp_registration_and_last_owner_release_advance_inventory_revision() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let initial = Arc::clone(&cache.temp_files.lock().unwrap().revision);
let registration = cache.register_active_temp_path(&directory.path().join("segment.ts.tmp.test"))?;
let registered = Arc::clone(&cache.temp_files.lock().unwrap().revision);
assert!(!Arc::ptr_eq(&initial, &registered));
let last_owner = Arc::clone(&registration);
drop(registration);
assert!(Arc::ptr_eq(&registered, &cache.temp_files.lock().unwrap().revision));
drop(last_owner);
assert!(!Arc::ptr_eq(&registered, &cache.temp_files.lock().unwrap().revision));
assert!(cache.temp_files.lock().unwrap().active_files.is_empty());
Ok(())
}
#[tokio::test]
async fn attaching_a_budget_to_shared_staging_returns_an_error_and_preserves_its_owner() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let key = SegmentCacheKey::new(ProxySessionId("shared-staging".into()), 0, "ts");
let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?);
let staged = cache
.stage_temp_with_budget(
&key,
b"test".as_slice(),
Instant::now() + std::time::Duration::from_secs(2),
Some(Arc::clone(&budget)),
)
.await?;
let error = staged.clone().with_write_budget(None).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
assert_eq!(tokio::fs::read(&staged.path).await?, b"test");
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20);
cache.commit_staged(&key, staged).await?;
drop(budget);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 4);
Ok(())
}
#[tokio::test]
async fn inventory_scan_retries_when_a_reserved_temp_file_is_registered_after_its_snapshot() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = Arc::new(HlsSegmentCache::with_cache_path(directory.path()));
cache.update_cache_limits(60, 60);
let key = SegmentCacheKey::new(ProxySessionId("inventory-race".into()), 0, "ts");
cache.write_bytes_and_commit(&key, &[1; 20]).await?;
let budget = Arc::new(cache.reserve_revision_peak(&key, 20).await?);
cache.invalidate_capacity_accounting();
let (reached, resume) = cache.pause_startup_write(StartupWritePhase::AfterInventorySnapshot);
let scan_cache = Arc::clone(&cache);
let id = key.proxy_session_id().clone();
let scan = tokio::spawn(async move { scan_cache.capacity_usage(&id).await });
tokio::time::timeout(std::time::Duration::from_secs(2), reached).await.unwrap().unwrap();
let staged = cache
.stage_temp_with_budget(
&key,
&[2; 20][..],
Instant::now() + std::time::Duration::from_secs(2),
Some(Arc::clone(&budget)),
)
.await?;
resume.send(()).unwrap();
let usage = tokio::time::timeout(std::time::Duration::from_secs(2), scan).await.unwrap().unwrap()?;
cache.remove_staged(staged).await?;
drop(budget);
assert_eq!(usage.global_bytes, 20, "reserved staging bytes must not enter the committed inventory");
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 20);
let next = cache.reserve_revision_peak(&key, 30).await?;
drop(next);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
Ok(())
}
#[tokio::test]
async fn revision_peak_and_legacy_cache_share_the_global_and_session_quota() -> io::Result<()> {
let directory = tempfile::tempdir()?;
@@ -54,3 +130,375 @@ async fn processed_revision_retries_and_ranges_preserve_bytes_across_cache_repla
assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0);
Ok(())
}
// These are the two real disk-cache transitions from debug/bug.log. Sparse
// fixtures preserve the exact byte counts without allocating a GiB of RAM/disk.
#[tokio::test]
async fn logged_normal_commit_consumes_its_own_peak_reservation() -> io::Result<()> {
logged_peak_case(9_994_832).await
}
#[tokio::test]
async fn logged_revision_publication_consumes_its_remaining_peak_reservation() -> io::Result<()> {
logged_peak_case(8_052_416).await
}
async fn logged_peak_case(size: u64) -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
cache.update_cache_limits(1_073_741_824, 1_073_741_824);
let key = SegmentCacheKey::new(ProxySessionId("logged-session".into()), 84, "ts");
let existing = SegmentCacheKey::new(key.proxy_session_id().clone(), 0, "ts");
let path = cache.object_path(&existing);
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
let file = tokio::fs::File::create(path).await?;
file.set_len(1_040_683_024).await?;
drop(file);
let reservation = std::sync::Arc::new(cache.reserve_revision_peak(&key, 3 * size).await?);
let staged_path = cache.object_path(&key).with_extension("ts.tmp.regression");
let file = tokio::fs::File::create(&staged_path).await?;
file.set_len(size).await?;
drop(file);
let staged = cache.adopt_staged_file(staged_path, size)?.with_write_budget(Some(reservation.clone()))?;
let committed = cache.commit_staged(&key, staged).await?;
let store = crate::SegmentRevisionStore::default();
let revision = store.create(key.proxy_session_id().clone(), 84, crate::SegmentRevisionKind::Processed)?;
let published = cache
.publish_processed_revision_with_budget(
&revision.revision().key,
&committed.path,
Instant::now() + std::time::Duration::from_secs(5),
Some(reservation.clone()),
)
.await?;
assert_eq!(published.size, size);
drop(reservation);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 1_040_683_024 + 2 * size);
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
Ok(())
}
#[tokio::test]
async fn processed_reservation_release_wakes_before_and_after_wait_registration() -> io::Result<()> {
for registered in [false, true] {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let key = SegmentCacheKey::new(ProxySessionId("release".into()), 0, "ts");
let reservation = cache.reserve_revision_peak(&key, 30).await?;
assert!(!reservation.unaccounted_writes);
let revision = cache.capacity_revision();
let waiting = cache.wait_for_capacity_change(&revision);
tokio::pin!(waiting);
if registered {
assert!(futures::poll!(waiting.as_mut()).is_pending());
}
drop(reservation);
tokio::time::timeout(std::time::Duration::from_secs(1), waiting)
.await
.expect("releasing only a processed reservation must wake the waiter");
}
Ok(())
}
#[tokio::test]
async fn concurrent_peak_owners_cannot_consume_each_others_session_or_global_budget() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = std::sync::Arc::new(HlsSegmentCache::with_cache_path(directory.path()));
cache.update_cache_limits(180, 120);
let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(4));
let mut tasks = Vec::new();
for (session, seq) in [("one", 0), ("one", 1), ("two", 0)] {
let cache = cache.clone();
let barrier = barrier.clone();
tasks.push(tokio::spawn(async move {
let key = SegmentCacheKey::new(ProxySessionId(session.into()), seq, "ts");
let peak = std::sync::Arc::new(cache.reserve_revision_peak(&key, 60).await?);
barrier.wait().await;
let staged = cache
.stage_temp_with_budget(
&key,
&[7; 20][..],
Instant::now() + std::time::Duration::from_secs(2),
Some(peak.clone()),
)
.await?;
let metadata = cache.commit_staged(&key, staged).await?;
let store = crate::SegmentRevisionStore::default();
let owner = store.create(key.proxy_session_id().clone(), seq, crate::SegmentRevisionKind::Processed)?;
cache
.publish_processed_revision_with_budget(
&owner.revision().key,
&metadata.path,
Instant::now() + std::time::Duration::from_secs(2),
Some(peak),
)
.await?;
Ok::<_, io::Error>(())
}));
}
barrier.wait().await;
for task in tasks {
task.await??;
}
let usage = cache.capacity_usage(&ProxySessionId("one".into())).await?;
assert_eq!(usage, HlsCacheCapacityUsage { session_bytes: 80, global_bytes: 120 });
assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("one".into())), 0);
assert_eq!(cache.remaining_revision_bytes(&ProxySessionId("two".into())), 0);
Ok(())
}
#[tokio::test]
async fn old_write_cleanup_cannot_delete_a_newer_fetch_attempt() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let key = SegmentCacheKey::new(ProxySessionId("replace".into()), 0, "ts");
let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
let staged = cache
.stage_temp_with_budget(
&key,
b"old".as_slice(),
Instant::now() + std::time::Duration::from_secs(2),
Some(old.clone()),
)
.await?;
cache.commit_staged(&key, staged).await?;
let new = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
let staged = cache
.stage_temp_with_budget(
&key,
b"new".as_slice(),
Instant::now() + std::time::Duration::from_secs(2),
Some(new.clone()),
)
.await?;
cache.commit_staged(&key, staged).await?;
let store = crate::SegmentRevisionStore::default();
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
let error = cache
.publish_processed_revision_with_budget(
&revision.revision().key,
&cache.object_path(&key),
Instant::now() + std::time::Duration::from_secs(2),
Some(old.clone()),
)
.await
.expect_err("an old startup attempt cannot publish a replacement owned by a newer attempt");
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
assert!(cache.metadata(&revision.revision().key).await?.is_none());
cache.delete_write_owned(&key, &old).await?;
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new");
cache.delete_write_owned(&key, &new).await?;
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0);
Ok(())
}
#[tokio::test]
async fn pinned_revisions_remain_accounted_after_inventory_reconciliation_and_deletion() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let metrics = std::sync::Arc::new(crate::HlsCacheMetrics::default());
cache.install_revision_metrics(&metrics);
let key = SegmentCacheKey::new(ProxySessionId("pinned-scan".into()), 0, "ts");
cache.write_bytes_and_commit(&key, &[1; 20]).await?;
let store = crate::SegmentRevisionStore::default();
let owner = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
let pin = cache.pin_revision_file(&cache.object_path(&owner.revision().key))?;
*owner.revision().file_pin.lock().map_err(|_| io::Error::other("file pin"))? = Some(pin);
cache
.publish_processed_revision(
&owner.revision().key,
&cache.object_path(&key),
Instant::now() + std::time::Duration::from_secs(2),
)
.await?;
drop(cache.capacity_invalidation_guard());
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 40);
assert_eq!(metrics.snapshot().revision_disk_bytes, cache.scan_revision_disk_bytes().await?);
drop(owner);
cache.delete_session_dir(key.proxy_session_id()).await?;
assert_eq!(metrics.snapshot().revision_disk_bytes, 0);
assert_eq!(cache.scan_revision_disk_bytes().await?, 0);
Ok(())
}
#[tokio::test]
async fn failed_partial_commit_cleanup_retries_without_deleting_a_replacement() -> io::Result<()> {
for replace in [false, true] {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
let key = SegmentCacheKey::new(ProxySessionId("cleanup-retry".into()), 0, "ts");
let old = std::sync::Arc::new(cache.reserve_revision_peak(&key, 30).await?);
let staged = cache
.stage_temp_with_budget(
&key,
b"old".as_slice(),
Instant::now() + std::time::Duration::from_secs(2),
Some(old.clone()),
)
.await?;
cache.commit_staged(&key, staged).await?;
cache.fail_next_cache_deletes(cache.object_path(&key), 1);
assert!(cache.delete_write_owned(&key, &old).await.is_err());
drop(old);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 3);
if replace {
cache.write_bytes_and_commit(&key, b"new").await?;
}
cache.retry_abandoned_write_deletions().await;
if replace {
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, b"new");
} else {
assert!(cache.metadata(&key).await?.is_none());
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 0);
}
cache.retry_abandoned_write_deletions().await;
}
Ok(())
}
#[tokio::test]
async fn cancelled_staging_keeps_its_peak_until_pending_file_io_is_joined() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
cache.update_cache_limits(20, 20);
let key = SegmentCacheKey::new(ProxySessionId("pending-io".into()), 0, "ts");
let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?);
let path = cache.object_path(&key).with_extension("ts.tmp.pending");
tokio::fs::create_dir_all(path.parent().unwrap()).await?;
let staged = cache.adopt_staged_file(path.clone(), 0)?.with_write_budget(Some(budget.clone()))?;
let writer = staged.track_writer(tokio::fs::File::create(&path).await?);
let mut in_flight_io = writer.lock().await;
in_flight_io.as_mut().unwrap().write_all(b"pending").await?;
drop(budget);
let revision = cache.capacity_revision();
drop(staged); // Models cancellation while the producer still owns its I/O handle.
let released = cache.wait_for_capacity_change(&revision);
tokio::pin!(released);
assert!(futures::poll!(released.as_mut()).is_pending());
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 20);
drop(in_flight_io);
tokio::time::timeout(std::time::Duration::from_secs(2), released).await.unwrap();
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
assert!(!tokio::fs::try_exists(path).await?);
assert!(!cache.has_active_temp_files());
Ok(())
}
#[tokio::test]
async fn cancelled_revision_link_keeps_its_peak_until_the_owned_link_and_unlink_finish() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
cache.update_cache_limits(20, 20);
let key = SegmentCacheKey::new(ProxySessionId("pending-link".into()), 0, "ts");
let budget = std::sync::Arc::new(cache.reserve_revision_peak(&key, 20).await?);
let staged = cache
.stage_temp_with_budget(
&key,
b"0123456789".as_slice(),
Instant::now() + std::time::Duration::from_secs(5),
Some(budget.clone()),
)
.await?;
cache.commit_staged(&key, staged).await?;
let store = crate::SegmentRevisionStore::default();
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink);
let source = cache.object_path(&key);
let publish = cache.publish_processed_revision_with_budget(
&revision.revision().key,
&source,
Instant::now() + std::time::Duration::from_secs(5),
Some(budget),
);
let mut publish = Box::pin(publish);
tokio::select! {
entered = entered => entered.unwrap(),
result = &mut publish => panic!("publication bypassed its link checkpoint: {result:?}"),
}
drop(publish);
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10);
let token = cache.capacity_revision();
resume.send(()).unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), cache.wait_for_capacity_change(&token)).await.unwrap();
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 10);
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
assert!(!cache.has_active_temp_files());
assert!(cache.metadata(&revision.revision().key).await?.is_none());
assert_eq!(tokio::fs::read(source).await?, b"0123456789");
Ok(())
}
#[tokio::test]
async fn concurrent_cache_replacement_cannot_change_a_processed_publication_source_or_its_accounted_size(
) -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
cache.update_cache_limits(64, 64);
let key = SegmentCacheKey::new(ProxySessionId("publication-source".into()), 0, "ts");
cache.write_bytes_and_commit(&key, b"old").await?;
let store = crate::SegmentRevisionStore::default();
let revision = store.create(key.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?;
let source = cache.object_path(&key);
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeRevisionLink);
let publication = cache.publish_processed_revision(
&revision.revision().key,
&source,
Instant::now() + std::time::Duration::from_secs(5),
);
tokio::pin!(publication);
tokio::select! {
reached = entered => reached.unwrap(),
result = &mut publication => panic!("publication bypassed the source checkpoint: {result:?}"),
}
let replacement = cache.write_bytes_and_commit(&key, b"larger-new-cache-body");
tokio::pin!(replacement);
assert!(futures::poll!(replacement.as_mut()).is_pending());
resume.send(()).unwrap();
let (published, replaced) =
tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(publication, replacement) })
.await
.expect("source lease releases before publication commit reacquires admission");
let published = published?;
let replaced = replaced?;
assert_eq!(published.size, 3);
assert_eq!(tokio::fs::read(published.path).await?, b"old");
assert_eq!(tokio::fs::read(replaced.path).await?, b"larger-new-cache-body");
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, replaced.size + 3);
assert_eq!(cache.revision_disk_bytes(), cache.scan_revision_disk_bytes().await?);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
Ok(())
}
#[tokio::test]
async fn shrinking_replacement_keeps_old_bytes_charged_until_rename_finishes() -> io::Result<()> {
let directory = tempfile::tempdir()?;
let cache = HlsSegmentCache::with_cache_path(directory.path());
cache.update_cache_limits(110, 110);
let key = SegmentCacheKey::new(ProxySessionId("shrinking".into()), 0, "ts");
let other = SegmentCacheKey::new(ProxySessionId("other-session".into()), 0, "ts");
cache.write_bytes_and_commit(&key, &[1; 100]).await?;
let (entered, resume) = cache.pause_startup_write(super::super::StartupWritePhase::BeforeCacheRename);
let replacement = cache.write_bytes_and_commit(&key, &[2; 10]);
tokio::pin!(replacement);
tokio::select! {
reached = entered => reached.unwrap(),
result = &mut replacement => panic!("replacement bypassed the rename checkpoint: {result:?}"),
}
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [1; 100]);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.session_bytes, 100);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 10);
assert!(cache.reserve_revision_peak(&key, 1).await.is_err());
assert!(cache.reserve_revision_peak(&other, 1).await.is_err());
resume.send(()).unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), replacement).await.unwrap()?;
assert_eq!(tokio::fs::read(cache.object_path(&key)).await?, [2; 10]);
assert_eq!(cache.capacity_usage(key.proxy_session_id()).await?.global_bytes, 10);
assert_eq!(cache.remaining_revision_bytes(key.proxy_session_id()), 0);
let released = cache.reserve_revision_peak(&other, 100).await?;
drop(released);
Ok(())
}
+123 -18
View File
@@ -27,9 +27,12 @@ impl HlsRevisionDiskReservation {
impl Drop for HlsRevisionDiskReservation {
fn drop(&mut self) {
let mut capacity = self.capacity.lock_unpoisoned();
capacity.revision_reservations.remove(&self.id);
let released = capacity.revision_reservations.remove(&self.id).is_some();
capacity.reserved_spools.retain(|_, owner| *owner != self.id);
if self.unaccounted_writes {
capacity.initialized = false;
}
if released || self.unaccounted_writes {
capacity.revision = Arc::new(CapacityRevision);
}
drop(capacity);
@@ -48,6 +51,14 @@ impl HlsSegmentCache {
bytes: u64,
) -> io::Result<HlsRevisionDiskReservation> {
let _admission = self.capacity_admission_gate.lock().await;
self.reserve_revision_peak_inner(key, bytes).await
}
pub(super) async fn reserve_revision_peak_inner<K: HlsCacheObjectKey>(
&self,
key: &K,
bytes: u64,
) -> io::Result<HlsRevisionDiskReservation> {
let root = self.cache_path_snapshot();
self.ensure_capacity_initialized(&root.path).await?;
let session = key.session_path_component();
@@ -77,19 +88,20 @@ impl HlsSegmentCache {
HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
));
}
let id = loop {
let id = fastrand::u64(..);
if !capacity.revision_reservations.contains_key(&id) {
break id;
}
};
let id =
capacity.next_write_id.checked_add(1).ok_or_else(|| io::Error::other("cache write identity exhausted"))?;
capacity.next_write_id = id;
capacity.revision_reservations.insert(id, (session, bytes));
capacity.revision = Arc::new(CapacityRevision);
drop(capacity);
self.capacity_changed.notify_waiters();
Ok(HlsRevisionDiskReservation {
id,
capacity: Arc::clone(&self.capacity),
changed: Arc::clone(&self.capacity_changed),
unaccounted_writes: false,
generation: root.generation,
cache_path: root.path,
})
}
@@ -101,18 +113,44 @@ impl HlsSegmentCache {
}
pub async fn create_revision_spool<K: HlsCacheObjectKey>(&self, key: &K) -> io::Result<(File, HlsRevisionFilePin)> {
self.create_revision_spool_with_budget(key, None).await
}
pub(crate) async fn create_revision_spool_with_budget<K: HlsCacheObjectKey>(
&self,
key: &K,
budget: Option<&HlsRevisionDiskReservation>,
) -> io::Result<(File, HlsRevisionFilePin)> {
let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await;
let root = self.cache_path_snapshot();
if budget.is_some_and(|budget| !Arc::ptr_eq(&budget.generation, &root.generation)) {
return Err(io::Error::new(io::ErrorKind::Interrupted, "revision reservation root retired"));
}
self.ensure_cache_root_marker_for(&root.path).await?;
let path = Self::path_for_key_in(&root.path, key);
let pin = self.pin_revision_file(&path)?;
let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?;
fs::create_dir_all(parent).await?;
let file = OpenOptions::new().write(true).create_new(true).open(&path).await?;
Ok((file, pin))
let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?.to_path_buf();
let mut mutation = self.begin_capacity_mutation(&root.path, &path, key.session_path_component()).await;
if let Some(budget) = budget {
self.register_reserved_spool(path.clone(), budget);
}
super::staging::run_owned_cache_operation(
Arc::clone(&self.owned_operation_permits),
"revision-create",
async move {
fs::create_dir_all(parent).await?;
let file = OpenOptions::new().write(true).create_new(true).open(&path).await?;
mutation.finish_mutation(None);
drop(commit_gate);
Ok((file, pin))
},
)
.await
}
pub async fn delete_orphan_revision_files(&self, cutoff: SystemTime) -> io::Result<usize> {
let root = self.cache_path_snapshot();
self.ensure_capacity_initialized(&root.path).await?;
let Some(mut sessions) = skip_not_found(fs::read_dir(&root.path).await)? else { return Ok(0) };
let mut deleted = 0usize;
while let Some(session) = sessions.next_entry().await? {
@@ -154,23 +192,90 @@ impl HlsSegmentCache {
key: &K,
source: &Path,
deadline: Instant,
) -> io::Result<CachedSegmentMetadata> {
self.publish_processed_revision_with_budget(key, source, deadline, None).await
}
#[allow(clippy::too_many_lines, reason = "Keeps source ownership, peak admission and immutable linking together.")]
pub(crate) async fn publish_processed_revision_with_budget<K: HlsCacheObjectKey>(
&self,
key: &K,
source: &Path,
deadline: Instant,
budget: Option<Arc<HlsRevisionDiskReservation>>,
) -> io::Result<CachedSegmentMetadata> {
let root = self.cache_path_snapshot();
let staged_path = Self::temp_path_for_key(key, &root.path);
let size = fs::metadata(source).await?.len();
let checks_source_owner = budget.is_some();
let budget = match budget {
Some(budget) => budget,
None => Arc::new(self.reserve_revision_peak_reclaiming(key, size).await?),
};
let (size, source_lease) = loop {
if Instant::now() >= deadline {
return Err(io::Error::new(io::ErrorKind::TimedOut, "revision publication timed out"));
}
self.ensure_write_budget_available(key, &budget, size).await?;
// Preserve admission -> path-mutation ordering. A replacement must
// not change the source between its size check and the hardlink.
let admission = self.capacity_admission_gate.lock().await;
let source_lease = self.begin_capacity_mutation(&root.path, source, key.session_path_component()).await;
let actual_size = fs::metadata(source).await?.len();
let remaining = {
let capacity = self.capacity.lock_unpoisoned();
if checks_source_owner && capacity.write_owners.get(source) != Some(&budget.id) {
return Err(io::Error::new(io::ErrorKind::Interrupted, "revision source write replaced"));
}
capacity.revision_reservations.get(&budget.id).map_or(0, |(_, bytes)| *bytes)
};
drop(admission);
if actual_size <= remaining {
break (actual_size, source_lease);
}
drop(source_lease);
self.ensure_write_budget_available(key, &budget, actual_size).await?;
};
let budget = Some(budget);
let registration = self.register_active_temp_path(&staged_path)?;
if fs::hard_link(source, &staged_path).await.is_ok() {
let size = fs::metadata(&staged_path).await?.len();
let staged = StagedCacheObject::registered(staged_path, size, Arc::clone(&root.generation), registration);
let staged =
StagedCacheObject::registered(staged_path.clone(), size, Arc::clone(&root.generation), registration)
.with_write_budget(budget.clone())?;
let source_path = source.to_path_buf();
#[cfg(test)]
let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeRevisionLink);
// A Tokio filesystem call can finish after its caller is cancelled.
// Retain staging and its budget until the link has a definite result.
let (staged, fallback_source) = super::staging::run_owned_cache_operation(
Arc::clone(&self.owned_operation_permits),
"revision-link",
async move {
#[cfg(test)]
if let Some(checkpoint) = checkpoint {
checkpoint.wait().await?;
}
let fallback = if fs::hard_link(&source_path, &staged_path).await.is_ok() {
None
} else {
// Opening under the same source lease also freezes the
// inode used by a copy fallback after replacement resumes.
Some(File::open(source_path).await?)
};
drop(source_lease);
Ok((staged, fallback))
},
)
.await?;
if let Some(file) = fallback_source {
self.remove_staged(staged).await?;
let staged = self.stage_temp_with_budget(key, file, deadline, budget).await?;
self.commit_staged(key, staged).await
} else {
drop(registration);
let file = File::open(source).await?;
let staged = self.stage_temp_with_deadline(key, file, deadline).await?;
self.commit_staged(key, staged).await
}
}
/// Revision file bytes as of the last completed capacity scan.
/// Current revision file bytes after completed accounting mutations.
pub fn revision_disk_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().revision_bytes }
/// Walks the cache tree for the current revision file bytes.
+100 -14
View File
@@ -34,7 +34,20 @@ impl StagedCacheObject {
cache_path_generation: Arc<CachePathGeneration>,
registration: Arc<ActiveTempFileRegistration>,
) -> Self {
Self { path, size, cache_path_generation, _registration: registration }
Self {
cleanup: Arc::new(super::StagedFileCleanup {
path: path.clone(),
registration: Arc::clone(&registration),
budget: None,
completed: std::sync::atomic::AtomicBool::new(false),
writer: std::sync::Mutex::new(None),
}),
path,
size,
cache_path_generation,
_registration: registration,
write_budget: None,
}
}
}
@@ -118,9 +131,10 @@ impl HlsSegmentCache {
G: Send + 'static,
{
let staged_path = staged.path.clone();
let cleanup = Arc::clone(&staged.cleanup);
let result = self.commit_staged_inner(key, staged, guard).await;
if result.is_err() {
remove_temp_file_after_error(&staged_path, "commit").await;
cleanup.completed.store(remove_temp_file_after_error(&staged_path, "commit").await, Ordering::Release);
}
result
}
@@ -164,13 +178,14 @@ impl HlsSegmentCache {
};
fs::create_dir_all(parent).await?;
let reservation = self
.prepare_capacity_replacement(
.prepare_capacity_replacement_with_budget(
&cache_path.path,
&final_path,
key.session_path_component(),
staged.size,
self.max_cache_bytes.load(Ordering::Acquire),
self.max_session_bytes.load(Ordering::Acquire),
staged.write_budget.clone(),
)
.await;
let reservation = match reservation {
@@ -265,18 +280,25 @@ impl HlsSegmentCache {
drop(admission_gate);
let staged_path = staged.path.clone();
let committed_size = staged.size;
#[cfg(test)]
let checkpoint = self.take_startup_write_checkpoint(super::StartupWritePhase::BeforeCacheRename);
// Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This
// keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem
// operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow
// that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation.
return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move {
let mut reservation = reservation;
#[cfg(test)]
if let Some(checkpoint) = checkpoint {
checkpoint.wait().await?;
}
reservation.mark_filesystem_mutation_started();
if let Err(err) = fs::rename(&staged_path, &final_path).await {
remove_temp_file_after_error(&staged_path, "rename").await;
return Err(err);
}
reservation.finish_replacement();
staged.cleanup.completed.store(true, Ordering::Release);
drop(staged);
drop(commit_gate);
Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard))
@@ -286,11 +308,17 @@ impl HlsSegmentCache {
}
pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> {
staged.finish_writer().await;
#[cfg(test)]
self.check_cache_delete_test_failure(&staged.path)?;
let result = match fs::remove_file(&staged.path).await {
Ok(()) => Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(err),
};
if result.is_ok() {
staged.cleanup.completed.store(true, Ordering::Release);
}
drop(staged);
result
}
@@ -315,6 +343,20 @@ impl HlsSegmentCache {
reader: &mut R,
deadline: Option<Instant>,
) -> io::Result<StagedCacheObject>
where
K: HlsCacheObjectKey,
R: AsyncRead + Unpin,
{
self.stage_temp_inner_with_budget(key, reader, deadline, None).await
}
pub(super) async fn stage_temp_inner_with_budget<K, R>(
&self,
key: &K,
reader: &mut R,
deadline: Option<Instant>,
budget: Option<Arc<super::HlsRevisionDiskReservation>>,
) -> io::Result<StagedCacheObject>
where
K: HlsCacheObjectKey,
R: AsyncRead + Unpin,
@@ -322,6 +364,10 @@ impl HlsSegmentCache {
if deadline.is_some_and(|deadline| deadline <= Instant::now()) {
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
}
let budget = match budget {
Some(budget) => budget,
None => Arc::new(self.reserve_revision_peak_reclaiming(key, 0).await?),
};
let cache_path = self.cache_path_snapshot();
self.ensure_cache_root_marker_for(&cache_path.path).await?;
let final_path = Self::path_for_key_in(&cache_path.path, key);
@@ -330,28 +376,61 @@ impl HlsSegmentCache {
};
fs::create_dir_all(parent).await?;
let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?;
let (temp_path, temp_file, registration) = self.create_temp_file(key, &cache_path).await?;
let cleanup = StagedCacheObject::registered(
temp_path.clone(),
0,
Arc::clone(&cache_path.generation),
Arc::clone(&registration),
)
.with_write_budget(Some(Arc::clone(&budget)))?;
let writer = cleanup.track_writer(temp_file);
if deadline.is_some_and(|deadline| deadline <= Instant::now()) {
drop(temp_file);
remove_temp_file_after_error(&temp_path, "expired-write").await;
cleanup.finish_writer().await;
cleanup
.cleanup
.completed
.store(remove_temp_file_after_error(&temp_path, "expired-write").await, Ordering::Release);
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
}
let copy = async {
let mut writer = writer.lock().await;
let temp_file = writer.as_mut().ok_or_else(|| io::Error::other("staging writer already closed"))?;
let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire);
let mut limited = reader.take(max_object_bytes.saturating_add(1));
let size = tokio::io::copy(&mut limited, &mut temp_file).await?;
let mut size = 0u64;
let mut buffer = [0u8; 16 * 1024];
loop {
let read = limited.read(&mut buffer).await?;
if read == 0 {
break;
}
let next_size = size.saturating_add(u64::try_from(read).map_err(io::Error::other)?);
if next_size > max_object_bytes {
return Err(cache_object_limit_error(max_object_bytes));
}
// Admission precedes every filesystem write, including headerless
// bodies and standalone cache users without a startup transaction.
self.ensure_write_budget_available(key, &budget, next_size).await?;
temp_file.write_all(&buffer[..read]).await?;
size = next_size;
}
if size > max_object_bytes {
return Err(cache_object_limit_error(max_object_bytes));
}
temp_file.flush().await?;
drop(temp_file);
writer.take();
Ok::<u64, io::Error>(size)
};
let copy_result = if let Some(deadline) = deadline {
if let Ok(result) = timeout_at(deadline, copy).await {
result
} else {
remove_temp_file_after_error(&temp_path, "timed-out-write").await;
cleanup.finish_writer().await;
cleanup
.cleanup
.completed
.store(remove_temp_file_after_error(&temp_path, "timed-out-write").await, Ordering::Release);
return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out"));
}
} else {
@@ -359,10 +438,16 @@ impl HlsSegmentCache {
};
match copy_result {
Ok(size) => {
Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration))
let mut staged = cleanup;
staged.size = size;
Ok(staged)
}
Err(err) => {
remove_temp_file_after_error(&temp_path, "failed-write").await;
cleanup.finish_writer().await;
cleanup
.cleanup
.completed
.store(remove_temp_file_after_error(&temp_path, "failed-write").await, Ordering::Release);
Err(err)
}
}
@@ -411,12 +496,13 @@ impl HlsSegmentCache {
}
}
async fn remove_temp_file_after_error(path: &Path, operation: &'static str) {
async fn remove_temp_file_after_error(path: &Path, operation: &'static str) -> bool {
match fs::remove_file(path).await {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Ok(()) => true,
Err(error) if error.kind() == io::ErrorKind::NotFound => true,
Err(error) => {
warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind());
false
}
}
}
+69
View File
@@ -0,0 +1,69 @@
use super::HlsSegmentCache;
use crate::sync_ext::MutexExt;
use std::{
io,
path::{Path, PathBuf},
};
use tokio::sync::oneshot;
#[derive(Clone, Copy, Eq, PartialEq)]
pub(crate) enum StartupWritePhase {
AfterInventorySnapshot,
BeforeNormalCommit,
AfterNormalCommit,
BeforeRevisionLink,
BeforeCacheRename,
AfterPublication,
}
pub(super) struct StartupWriteCheckpoint {
phase: StartupWritePhase,
entered: oneshot::Sender<()>,
resume: oneshot::Receiver<()>,
}
impl StartupWriteCheckpoint {
pub(super) async fn wait(self) -> io::Result<()> {
let _ = self.entered.send(());
self.resume.await.map_err(|_| io::Error::from_raw_os_error(28))
}
}
impl HlsSegmentCache {
pub(crate) fn pause_startup_write(&self, phase: StartupWritePhase) -> (oneshot::Receiver<()>, oneshot::Sender<()>) {
let (entered, reached) = oneshot::channel();
let (resume, released) = oneshot::channel();
*self.startup_test_checkpoint.lock_unpoisoned() =
Some(StartupWriteCheckpoint { phase, entered, resume: released });
(reached, resume)
}
pub(super) fn take_startup_write_checkpoint(&self, phase: StartupWritePhase) -> Option<StartupWriteCheckpoint> {
let mut hook = self.startup_test_checkpoint.lock_unpoisoned();
if hook.as_ref().is_some_and(|hook| hook.phase == phase) {
hook.take()
} else {
None
}
}
pub(crate) async fn startup_write_checkpoint(&self, phase: StartupWritePhase) -> io::Result<()> {
if let Some(hook) = self.take_startup_write_checkpoint(phase) {
hook.wait().await?;
}
Ok(())
}
pub(crate) fn fail_next_cache_deletes(&self, path: PathBuf, attempts: usize) {
self.delete_test_failures.lock_unpoisoned().insert(path, attempts);
}
pub(super) fn check_cache_delete_test_failure(&self, path: &Path) -> io::Result<()> {
let mut failures = self.delete_test_failures.lock_unpoisoned();
if let Some(remaining) = failures.get_mut(path).filter(|remaining| **remaining > 0) {
*remaining -= 1;
return Err(io::Error::new(io::ErrorKind::PermissionDenied, "injected cache deletion failure"));
}
Ok(())
}
}
+8 -18
View File
@@ -42,6 +42,8 @@ async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation()
session_component: "session".to_string(),
replacement: None,
filesystem_mutation_started: false,
write_budget: None,
transferred_bytes: 0,
capacity: Arc::clone(&cache.capacity),
changed: Arc::clone(&cache.capacity_changed),
};
@@ -79,6 +81,8 @@ async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() {
session_component: "session".to_string(),
replacement: None,
filesystem_mutation_started: false,
write_budget: None,
transferred_bytes: 0,
capacity: Arc::clone(&cache.capacity),
changed: Arc::clone(&cache.capacity_changed),
};
@@ -131,22 +135,6 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write
cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits");
cache.update_cache_limits(25, 25);
let first_staged = cache
.stage_temp_with_deadline(
&first_target,
&b"abcdefghij"[..],
tokio::time::Instant::now() + Duration::from_secs(5),
)
.await
.expect("first target stages");
let second_staged = cache
.stage_temp_with_deadline(
&second_target,
&b"klmnopqrst"[..],
tokio::time::Instant::now() + Duration::from_secs(5),
)
.await
.expect("second target stages");
let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel();
let (resume_first_tx, resume_first_rx) = oneshot::channel();
let reclaimer = Arc::new(PausingCapacityReclaimer {
@@ -158,11 +146,13 @@ async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_write
cache.install_capacity_reclaimer(&reclaimer);
let first_cache = Arc::clone(&cache);
let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await });
let first_task =
tokio::spawn(async move { first_cache.write_bytes_and_commit(&first_target, b"abcdefghij").await });
first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident");
let second_cache = Arc::clone(&cache);
let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await });
let second_task =
tokio::spawn(async move { second_cache.write_bytes_and_commit(&second_target, b"klmnopqrst").await });
for _ in 0..100 {
tokio::task::yield_now().await;
}
+393
View File
@@ -0,0 +1,393 @@
//! Ownership of peak bytes across staging, rename, and immutable publication.
use super::{
error::capacity_error, revisions::revision_reserved_bytes, ActiveTempFileRegistration, CacheCapacityPressure,
CapacityRevision, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsCacheCapacityRevision,
HlsCacheObjectKey, HlsRevisionDiskReservation, HlsSegmentCache, StagedCacheObject,
};
use crate::sync_ext::MutexExt;
use std::{
io,
path::PathBuf,
sync::{atomic::Ordering, Arc},
};
use tokio::{
fs,
io::{AsyncRead, AsyncWriteExt},
time::Instant,
};
type TrackedWriter = Arc<tokio::sync::Mutex<Option<fs::File>>>;
pub(super) struct StagedFileCleanup {
pub(super) path: PathBuf,
pub(super) registration: Arc<ActiveTempFileRegistration>,
pub(super) budget: Option<Arc<HlsRevisionDiskReservation>>,
pub(super) completed: std::sync::atomic::AtomicBool,
pub(super) writer: std::sync::Mutex<Option<TrackedWriter>>,
}
impl Drop for StagedFileCleanup {
fn drop(&mut self) {
if self.completed.load(Ordering::Acquire) {
return;
}
let path = self.path.clone();
let registration = Arc::clone(&self.registration);
let budget = self.budget.clone();
let writer = self.writer.lock_unpoisoned().clone();
// Keep both the file registration and peak budget until removal completes.
// A successful rename leaves no file at this unique temporary path.
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
runtime.spawn(async move {
finish_writer(writer).await;
if let Err(error) = fs::remove_file(path).await {
if error.kind() != io::ErrorKind::NotFound {
log::warn!("HLS staging cleanup deferred: error_kind={:?}", error.kind());
if let Some(budget) = &budget {
let mut capacity = budget.capacity.lock_unpoisoned();
capacity.initialized = false;
capacity.revision = Arc::new(CapacityRevision);
}
}
}
drop(registration);
drop(budget);
});
}
}
}
async fn finish_writer(writer: Option<TrackedWriter>) {
if let Some(writer) = writer {
let mut guard = writer.lock().await;
if let Some(mut file) = guard.take() {
// Tokio filesystem writes may outlive cancellation of write_all.
// Flushing the same handle joins that pending write before unlink.
let _ = file.flush().await;
}
}
}
impl StagedCacheObject {
pub(crate) fn track_writer(&self, file: fs::File) -> TrackedWriter {
let writer = Arc::new(tokio::sync::Mutex::new(Some(file)));
*self.cleanup.writer.lock_unpoisoned() = Some(Arc::clone(&writer));
writer
}
pub(super) async fn finish_writer(&self) {
let writer = self.cleanup.writer.lock_unpoisoned().clone();
finish_writer(writer).await;
}
pub(crate) fn with_write_budget(mut self, budget: Option<Arc<HlsRevisionDiskReservation>>) -> io::Result<Self> {
Arc::get_mut(&mut self.cleanup)
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "cannot change a shared staged file's budget"))?
.budget
.clone_from(&budget);
self.write_budget = budget;
Ok(self)
}
}
impl HlsSegmentCache {
pub(crate) async fn stage_temp_with_budget<K, R>(
&self,
key: &K,
mut reader: R,
deadline: Instant,
budget: Option<Arc<HlsRevisionDiskReservation>>,
) -> io::Result<StagedCacheObject>
where
K: HlsCacheObjectKey,
R: AsyncRead + Unpin,
{
self.stage_temp_inner_with_budget(key, &mut reader, Some(deadline), budget).await
}
pub(crate) async fn ensure_write_budget_available<K: HlsCacheObjectKey>(
&self,
key: &K,
budget: &HlsRevisionDiskReservation,
size: u64,
) -> io::Result<()> {
if !Arc::ptr_eq(&self.capacity, &budget.capacity)
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
{
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired write reservation"));
}
let remaining = self
.capacity
.lock_unpoisoned()
.revision_reservations
.get(&budget.id)
.filter(|(owner, _)| owner == &key.session_path_component())
.map(|(_, bytes)| *bytes)
.ok_or_else(|| io::Error::new(io::ErrorKind::Interrupted, "write reservation no longer current"))?;
if size > remaining {
if let Err(error) = self.grow_revision_peak(key, budget, size - remaining).await {
if let Some(capacity) = super::hls_cache_capacity_from_io(&error) {
let mut pressure = capacity.pressure();
pressure.staged_bytes = size;
return Err(capacity_error(
pressure,
HlsCacheCapacityReclaimOutcome {
protected_working_set_bytes: capacity.protected_working_set_bytes,
reclaimable_bytes: capacity.reclaimable_bytes,
..Default::default()
},
capacity.revision().clone(),
));
}
return Err(error);
}
}
Ok(())
}
pub(crate) async fn grow_revision_peak<K: HlsCacheObjectKey>(
&self,
key: &K,
budget: &HlsRevisionDiskReservation,
additional: u64,
) -> io::Result<()> {
let _admission = self.capacity_admission_gate.lock().await;
let root = self.cache_path_snapshot();
self.ensure_capacity_initialized(&root.path).await?;
let error = match self.try_grow_revision_peak(key, budget, additional) {
Ok(()) => return Ok(()),
Err(error) => error,
};
let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) };
let outcome = self
.reclaim_capacity(HlsCacheCapacityReclaimRequest {
proxy_session_id: key.proxy_session_id().clone(),
target_path: self.object_path(key),
required_session_bytes: capacity.required_session_bytes(),
required_global_bytes: capacity.required_global_bytes(),
staged_bytes: additional,
})
.await?;
self.ensure_capacity_initialized(&root.path).await?;
self.try_grow_revision_peak(key, budget, additional).map_err(|error| {
match super::hls_cache_capacity_from_io(&error) {
Some(capacity) => capacity_error(capacity.pressure(), outcome, capacity.revision().clone()),
None => error,
}
})
}
fn try_grow_revision_peak<K: HlsCacheObjectKey>(
&self,
key: &K,
budget: &HlsRevisionDiskReservation,
additional: u64,
) -> io::Result<()> {
if !Arc::ptr_eq(&self.capacity, &budget.capacity)
|| !Arc::ptr_eq(&budget.generation, &self.cache_path_snapshot().generation)
{
return Err(io::Error::new(io::ErrorKind::Interrupted, "foreign or retired cache write reservation"));
}
let session = key.session_path_component();
let mut capacity = self.capacity.lock_unpoisoned();
let (global, local) = revision_reserved_bytes(&capacity, &session);
let global = global.saturating_add(capacity.total_bytes);
let local = local.saturating_add(capacity.session_bytes.get(&session).copied().unwrap_or_default());
let global_limit = self.max_cache_bytes.load(Ordering::Acquire);
let local_limit = self.max_session_bytes.load(Ordering::Acquire);
if global.saturating_add(additional) > global_limit || local.saturating_add(additional) > local_limit {
return Err(capacity_error(
CacheCapacityPressure {
configured_session_bytes: local_limit,
configured_global_bytes: global_limit,
current_session_bytes: local,
current_global_bytes: global,
staged_bytes: additional,
required_session_bytes: local.saturating_add(additional).saturating_sub(local_limit),
required_global_bytes: global.saturating_add(additional).saturating_sub(global_limit),
},
HlsCacheCapacityReclaimOutcome::default(),
HlsCacheCapacityRevision(Arc::clone(&capacity.revision)),
));
}
let (owner, remaining) = capacity
.revision_reservations
.get_mut(&budget.id)
.filter(|(owner, _)| owner == &session)
.ok_or_else(|| io::Error::other("write reservation no longer current"))?;
let _ = owner;
*remaining = remaining.checked_add(additional).ok_or_else(|| io::Error::other("write budget overflow"))?;
capacity.revision = Arc::new(CapacityRevision);
drop(capacity);
self.capacity_changed.notify_waiters();
Ok(())
}
/// Admission of a complete fresh attempt; unlike replacement admission this
/// cannot credit a leftover stable file against the next attempt's peak.
pub(crate) async fn ensure_revision_peak_capacity<K: HlsCacheObjectKey>(
&self,
key: &K,
peak: u64,
) -> io::Result<()> {
let reservation = self.reserve_revision_peak_reclaiming(key, peak).await?;
drop(reservation);
Ok(())
}
pub(crate) async fn reserve_revision_peak_reclaiming<K: HlsCacheObjectKey>(
&self,
key: &K,
peak: u64,
) -> io::Result<HlsRevisionDiskReservation> {
// Reserve and reclaim under one admission gate; the reclaimer must never
// reenter admission. It only deletes already-owned files.
let _admission = self.capacity_admission_gate.lock().await;
let mut outcome = HlsCacheCapacityReclaimOutcome::default();
for attempt in 0..2 {
match self.reserve_revision_peak_inner(key, peak).await {
Ok(reservation) => return Ok(reservation),
Err(error) => {
let Some(capacity) = super::hls_cache_capacity_from_io(&error) else { return Err(error) };
if attempt == 1 {
return Err(capacity_error(capacity.pressure(), outcome, capacity.revision().clone()));
}
outcome = self
.reclaim_capacity(HlsCacheCapacityReclaimRequest {
proxy_session_id: key.proxy_session_id().clone(),
target_path: self.object_path(key),
required_session_bytes: capacity.required_session_bytes(),
required_global_bytes: capacity.required_global_bytes(),
staged_bytes: peak,
})
.await?;
}
}
}
unreachable!("bounded admission loop returns on second attempt")
}
/// Deletes only the stable file installed by this exact peak owner. The
/// path mutation ticket serializes this comparison with every later rename.
pub(crate) async fn delete_write_owned<K: HlsCacheObjectKey>(
&self,
key: &K,
budget: &HlsRevisionDiskReservation,
) -> io::Result<()> {
let path = Self::path_for_key_in(&budget.cache_path, key);
let mutation = self.begin_capacity_mutation(&budget.cache_path, &path, key.session_path_component()).await;
let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&budget.id);
if owned {
let result = self.delete_with_reservation(path.clone(), mutation).await;
if result.is_err() {
self.capacity
.lock_unpoisoned()
.abandoned_writes
.insert(path, (budget.id, key.session_path_component()));
}
result
} else {
Ok(())
}
}
pub(crate) async fn retry_abandoned_write_deletions(&self) {
let pending: Vec<_> = self
.capacity
.lock_unpoisoned()
.abandoned_writes
.iter()
.take(super::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN)
.map(|(path, owner)| (path.clone(), owner.clone()))
.collect();
let root = self.cache_path_snapshot();
for (path, (owner, session)) in pending {
if !path.starts_with(&root.path) {
self.capacity.lock_unpoisoned().abandoned_writes.remove(&path);
continue;
}
let mutation = self.begin_capacity_mutation(&root.path, &path, session).await;
let owned = self.capacity.lock_unpoisoned().write_owners.get(&path) == Some(&owner);
if owned {
let _ = self.delete_with_reservation(path, mutation).await;
} else {
self.capacity.lock_unpoisoned().abandoned_writes.remove(&path);
}
}
}
pub(crate) fn register_reserved_spool(&self, path: PathBuf, budget: &HlsRevisionDiskReservation) {
let mut capacity = self.capacity.lock_unpoisoned();
capacity.write_owners.insert(path.clone(), budget.id);
capacity.reserved_spools.insert(path, budget.id);
}
pub(crate) fn account_reserved_spool(
&self,
path: &std::path::Path,
size: u64,
budget: &HlsRevisionDiskReservation,
) -> io::Result<()> {
let mut capacity = self.capacity.lock_unpoisoned();
let (session, remaining) = capacity
.revision_reservations
.get_mut(&budget.id)
.ok_or_else(|| io::Error::other("spool reservation expired"))?;
let session = session.clone();
*remaining = remaining.checked_sub(size).ok_or_else(|| io::Error::other("spool exceeds peak budget"))?;
capacity.reserved_spools.remove(path);
capacity.total_bytes = capacity.total_bytes.saturating_add(size);
*capacity.session_bytes.entry(session).or_default() += size;
capacity.revision_bytes = capacity.revision_bytes.saturating_add(size);
super::capacity::update_revision_metrics(&capacity);
capacity.revision = Arc::new(CapacityRevision);
drop(capacity);
self.capacity_changed.notify_waiters();
Ok(())
}
pub(crate) fn install_revision_metrics(&self, metrics: &Arc<crate::HlsCacheMetrics>) {
let mut capacity = self.capacity.lock_unpoisoned();
capacity.revision_metrics = Some(Arc::downgrade(metrics));
super::capacity::update_revision_metrics(&capacity);
}
pub(crate) fn accounted_session_bytes(&self, session: &crate::ProxySessionId) -> u64 {
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
self.capacity.lock_unpoisoned().session_bytes.get(&key.session_path_component()).copied().unwrap_or_default()
}
pub(crate) fn accounted_global_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().total_bytes }
pub(crate) async fn session_revision_disk_bytes(&self, session: &crate::ProxySessionId) -> io::Result<u64> {
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
let mut entries = match fs::read_dir(self.cache_path_snapshot().path.join(key.session_path_component())).await {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0),
Err(error) => return Err(error),
};
let mut bytes = 0u64;
while let Some(entry) = entries.next_entry().await? {
if super::revisions::is_revision_cache_path(&entry.path()) && entry.file_type().await?.is_file() {
match entry.metadata().await {
Ok(metadata) => bytes = bytes.saturating_add(metadata.len()),
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
}
}
Ok(bytes)
}
pub(crate) fn remaining_revision_bytes(&self, session: &crate::ProxySessionId) -> u64 {
let key = super::SegmentCacheKey::new(session.clone(), 0, "ts");
revision_reserved_bytes(&self.capacity.lock_unpoisoned(), &key.session_path_component()).1
}
}
impl HlsRevisionDiskReservation {
/// Resolves cleanup against this write's original root, even after handoff.
pub(crate) fn object_path<K: HlsCacheObjectKey>(&self, key: &K) -> PathBuf {
HlsSegmentCache::path_for_key_in(&self.cache_path, key)
}
}
+2
View File
@@ -37,6 +37,7 @@ pub struct HlsGarbageCollector {
pending_cache_deletions: Arc<StdMutex<CacheDeletionQueueState>>,
access_leases: StdRwLock<Option<Weak<RwLock<HlsAccessLeaseStore>>>>,
run_once_gate: AsyncMutex<()>,
revision_store: StdRwLock<Option<Weak<super::SegmentRevisionStore>>>,
}
#[derive(Debug, Clone, Eq, PartialEq)]
@@ -112,6 +113,7 @@ mod deletion;
mod policy;
mod reclaim;
mod report;
mod revisions;
mod selection;
pub use collector::{build_rewrite_secret_fingerprint, exec_hls_cache_gc};
use deletion::{CacheDeletionQueueGeneration, CacheObjectDeletion};
+8
View File
@@ -52,6 +52,7 @@ impl HlsGarbageCollector {
pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())),
access_leases: StdRwLock::new(None),
run_once_gate: AsyncMutex::new(()),
revision_store: StdRwLock::new(None),
}
}
@@ -102,6 +103,11 @@ impl HlsGarbageCollector {
report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?;
let sessions = self.sessions.list_sessions().await;
if let Some(handle) = sessions.first() {
let id = handle.read().await.proxy_session_id.clone();
self.cache.capacity_usage(&id).await?;
}
self.cleanup_revisions(&sessions, &mut report).await?;
let mut active_session_ids = HashSet::with_capacity(sessions.len());
for session in &sessions {
active_session_ids.insert(session.read().await.proxy_session_id.clone());
@@ -122,6 +128,7 @@ impl HlsGarbageCollector {
session_deletions.persist(&mut report);
}
self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await;
let mut global_deletions = self.reserve_cache_deletion_batch();
self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await;
global_deletions.persist(&mut report);
@@ -131,6 +138,7 @@ impl HlsGarbageCollector {
self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await;
}
self.cleanup_revisions(&sessions, &mut report).await?;
report.cache_object_deletions_deferred = self.pending_cache_deletion_count();
self.record_report_metrics(&report);
if report.did_cleanup_or_invalidate() {
+58 -14
View File
@@ -4,8 +4,8 @@ use super::{
selection::{
capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment,
fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate,
remove_map_entry, remove_segment_entry, remove_stale_queue_entries, session_cache_size,
take_expired_transient_objects, total_sessions_cache_size, unprotected_unreferenced_map_ids,
remove_map_entry, remove_obsolete_deferred_heads, remove_segment_entry, remove_stale_queue_entries,
session_cache_size, take_expired_transient_objects, unprotected_unreferenced_map_ids,
},
CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome,
HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle,
@@ -41,6 +41,8 @@ impl HlsGarbageCollector {
self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await;
deletion_attempt_budget =
deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget));
let sessions = self.sessions.list_sessions().await;
self.cleanup_revisions(&sessions, &mut report).await?;
let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?;
let required_session_bytes = request
.required_session_bytes
@@ -73,6 +75,30 @@ impl HlsGarbageCollector {
Some(&request.target_path),
release_through,
);
let head = session.segments.first_key_value();
let reason = match head {
None => "no-candidates",
Some((seq, _)) if protected.segment_proxy_seqs.contains(seq) => "protected",
Some((_, entry)) => match entry.status {
super::SegmentCacheStatus::Ready { .. } => "reclaimable",
super::SegmentCacheStatus::CapacityDeferred { .. } if entry.origin_fetch_ref.is_some() => {
"deferred-origin-binding-retained"
}
super::SegmentCacheStatus::CapacityDeferred { .. } => "obsolete-deferred-head",
super::SegmentCacheStatus::Discovered => "not-cached",
super::SegmentCacheStatus::Queued { .. } => "queued",
super::SegmentCacheStatus::Fetching { .. } => "active-fetch",
super::SegmentCacheStatus::FailedRetryable { .. }
| super::SegmentCacheStatus::FailedPermanent { .. } => "failed-retention",
super::SegmentCacheStatus::Expired => "expired-retention",
},
};
log::debug!(
"HLS capacity FIFO: proxy_session={} head={:?} status={:?} blocker={reason}",
crate::safe_proxy_session_id(&session.proxy_session_id),
head.map(|(seq, _)| *seq),
head.map(|(_, entry)| &entry.status)
);
let evidence = capacity_reclamation_evidence(&session, &protected);
protected_working_set_bytes = evidence.protected_working_set_bytes;
reclaimable_bytes = evidence.reclaimable_bytes;
@@ -95,8 +121,8 @@ impl HlsGarbageCollector {
let _ = session.render_and_store_manifest(rendered_at_ms);
}
} else {
protected_working_set_bytes = after_pending.session_bytes;
session_lock_busy = true;
log::debug!("HLS capacity selection: proxy_session={} outcome=lock-conflict protected_working_set_known=false", crate::safe_proxy_session_id(&request.proxy_session_id));
}
drop(lease_guard);
if session_lock_busy {
@@ -112,22 +138,15 @@ impl HlsGarbageCollector {
}
}
self.cleanup_revisions(&sessions, &mut report).await?;
let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?;
let remaining_session_bytes = request
.required_session_bytes
.saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes));
if remaining_session_bytes > 0 {
return Ok(HlsCacheCapacityReclaimOutcome {
reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes),
reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes),
protected_working_set_bytes,
reclaimable_bytes,
});
}
let required_global_bytes = request
.required_global_bytes
.saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes));
if required_global_bytes > 0 {
if remaining_session_bytes == 0 && required_global_bytes > 0 {
let sessions = self.sessions.list_sessions().await;
let mut deletions = self.reserve_cache_deletion_batch();
self.collect_projected_global_reclamation(
@@ -139,9 +158,13 @@ impl HlsGarbageCollector {
)
.await;
deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await;
self.cleanup_revisions(&sessions, &mut report).await?;
}
let after = self.cache.capacity_usage(&request.proxy_session_id).await?;
if log::log_enabled!(log::Level::Debug) {
self.log_capacity_inventory(&request.proxy_session_id, &report).await;
}
Ok(HlsCacheCapacityReclaimOutcome {
reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes),
reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes),
@@ -150,6 +173,25 @@ impl HlsGarbageCollector {
})
}
/// Diagnostic I/O cannot change the result of capacity reclamation.
pub(super) async fn log_capacity_inventory(&self, id: &crate::ProxySessionId, report: &GarbageCollectionReport) {
let revision_bytes = match self.cache.session_revision_disk_bytes(id).await {
Ok(bytes) => bytes,
Err(error) => {
log::debug!(
"HLS capacity inventory unavailable: proxy_session={} error_kind={:?}",
crate::safe_proxy_session_id(id),
error.kind()
);
return;
}
};
let reserved_bytes = self.cache.remaining_revision_bytes(id);
log::debug!("HLS capacity inventory: proxy_session={} revision_bytes={revision_bytes} remaining_reserved_bytes={reserved_bytes} revision_deletes_planned={} revision_deletes_succeeded={} revision_deletes_failed={} revision_reclaimed_bytes={}",
crate::safe_proxy_session_id(id), report.revision_deletions_planned,
report.revision_deletions_succeeded, report.revision_deletions_failed, report.revision_bytes_reclaimed);
}
pub(super) fn collect_session_deletions(
session: &mut HlsSession,
cache: &HlsSegmentCache,
@@ -161,6 +203,7 @@ impl HlsGarbageCollector {
report.stale_queue_entries_removed =
report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session));
remove_obsolete_deferred_heads(session, cache, None, None, deletions);
let transient_before = session.transient.resources.len();
let protected = protected_set_for_cache_reclamation(session, cache, None, None);
session.transient.prune_expired_except(now_ms, &protected.key_resource_ids);
@@ -195,7 +238,8 @@ impl HlsGarbageCollector {
}
}
let mut session_size = session_cache_size(session);
let mut session_size =
cache.accounted_session_bytes(&session.proxy_session_id).max(session_cache_size(session));
while session_size > policy.cache_bytes_per_session && deletions.has_capacity() {
let protected = protected_set_for_cache_reclamation(session, cache, None, None);
let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids)
@@ -247,7 +291,7 @@ impl HlsGarbageCollector {
report: &mut GarbageCollectionReport,
deletions: &mut CacheDeletionBatch,
) {
let mut total_size = total_sessions_cache_size(sessions).await;
let mut total_size = self.cache.accounted_global_bytes();
let mut transient_readiness_advanced = HashSet::new();
let mut skipped_transient_sessions = HashSet::new();
+8 -1
View File
@@ -2,6 +2,11 @@ use super::ProxySessionId;
#[derive(Debug, Default, Clone, Eq, PartialEq)]
pub struct GarbageCollectionReport {
pub revision_deletions_planned: usize,
pub revision_deletions_succeeded: usize,
pub revision_deletions_failed: usize,
pub revision_bytes_reclaimed: u64,
pub revision_lock_conflicts: usize,
pub secret_cache_invalidated: bool,
pub secret_cache_invalidation_deferred: bool,
pub temp_files_deleted: usize,
@@ -29,7 +34,9 @@ impl GarbageCollectionReport {
}
pub fn did_cleanup_or_invalidate(&self) -> bool {
self.secret_cache_invalidated
self.revision_deletions_planned > 0
|| self.revision_deletions_failed > 0
|| self.secret_cache_invalidated
|| self.secret_cache_invalidation_deferred
|| self.temp_files_deleted > 0
|| self.orphan_session_dirs_deleted > 0
+141
View File
@@ -0,0 +1,141 @@
//! Shared, bounded retirement for regular GC and projected-capacity reclamation.
use super::{GarbageCollectionReport, HlsGarbageCollector, HlsSessionHandle};
use crate::{sync_ext::MutexExt, SegmentRevision, SegmentRevisionStore};
use std::{
collections::VecDeque,
io,
sync::{Arc, Weak},
};
struct RetirementBatch {
store: Arc<SegmentRevisionStore>,
pending: VecDeque<Arc<SegmentRevision>>,
}
impl Drop for RetirementBatch {
fn drop(&mut self) {
// Cancellation or a failed delete preserves retry ownership, including
// all later entries in the batch which have not reached the filesystem.
for revision in self.pending.drain(..) {
let _ = self.store.requeue_retired(revision);
}
}
}
impl HlsGarbageCollector {
pub(crate) fn install_revision_store(&self, store: &Arc<SegmentRevisionStore>) {
*self.revision_store.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::downgrade(store));
}
/// Caller already owns the GC gate. Never calls admission or reacquires GC.
#[allow(
clippy::too_many_lines,
reason = "One bounded lease/session snapshot followed by cancellation-safe filesystem retirement."
)]
pub(super) async fn cleanup_revisions(
&self,
sessions: &[HlsSessionHandle],
report: &mut GarbageCollectionReport,
) -> io::Result<()> {
self.cache.retry_abandoned_write_deletions().await;
let before = (
report.revision_deletions_planned,
report.revision_deletions_succeeded,
report.revision_deletions_failed,
report.revision_bytes_reclaimed,
);
let mut stores: Vec<_> = self
.revision_store
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.and_then(Weak::upgrade)
.into_iter()
.collect();
let leases = self
.access_leases
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.and_then(Weak::upgrade);
for handle in sessions {
let mut lease_guard = match &leases {
Some(leases) => {
if let Ok(guard) = leases.try_write() {
Some(guard)
} else {
report.revision_lock_conflicts += 1;
continue;
}
}
None => None,
};
let Ok(mut session) = handle.try_write() else {
report.revision_lock_conflicts += 1;
continue;
};
let id = session.proxy_session_id.clone();
let floor = session.segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq);
let crate::HlsSession { startup, segments, .. } = &mut *session;
if let Some(startup) = startup {
startup.revisions.retain(|seq, _| segments.contains_key(seq));
if !stores.iter().any(|store| Arc::ptr_eq(store, &startup.store)) {
stores.push(Arc::clone(&startup.store));
}
}
if let Some(leases) = &mut lease_guard {
leases.prune_revision_bindings(&id, floor);
}
}
let mut remaining = super::MAX_CACHE_DELETE_RETRIES_PER_RUN;
for store in stores {
if remaining == 0 {
break;
}
let pending = store.retire_unpinned_limit(remaining)?;
let mut batch = RetirementBatch { store, pending: pending.into() };
let attempts = batch.pending.len();
for _ in 0..attempts {
let Some(revision) = batch.pending.front().cloned() else { break };
report.revision_deletions_planned += 1;
remaining -= 1;
revision.file_pin.lock_unpoisoned().take();
let size = self.cache.metadata(&revision.key).await?.map_or(0, |metadata| metadata.size);
match self.cache.delete_if_inactive(&revision.key).await {
Ok(()) => {
batch.pending.pop_front();
report.revision_deletions_succeeded += 1;
report.revision_bytes_reclaimed = report.revision_bytes_reclaimed.saturating_add(size);
}
Err(error) => {
report.revision_deletions_failed += 1;
log::warn!(
"HLS revision delete deferred: proxy_session={} proxy_seq={} error_kind={:?}",
crate::safe_proxy_session_id(crate::HlsCacheObjectKey::proxy_session_id(&revision.key)),
revision.key.proxy_seq,
error.kind()
);
// Put failed entries at the end of this bounded batch;
// its Drop requeues them after successful work finishes.
batch.pending.rotate_left(1);
}
}
}
}
self.metrics.record_revision_gc(
report.revision_deletions_planned - before.0,
report.revision_deletions_succeeded - before.1,
report.revision_deletions_failed - before.2,
report.revision_bytes_reclaimed - before.3,
);
log::debug!(
"HLS revision GC: planned={} succeeded={} failed={} reclaimed_bytes={} lock_conflicts={}",
report.revision_deletions_planned,
report.revision_deletions_succeeded,
report.revision_deletions_failed,
report.revision_bytes_reclaimed,
report.revision_lock_conflicts
);
Ok(())
}
}
+25 -9
View File
@@ -8,15 +8,6 @@ use super::{
};
use std::{collections::HashSet, sync::Arc};
pub(super) async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 {
let mut total_size = 0_u64;
for session in sessions {
let session = session.read().await;
total_size = total_size.saturating_add(session_cache_size(&session));
}
total_size
}
pub(super) fn collect_projected_session_reclamation(
session: &mut HlsSession,
cache: &HlsSegmentCache,
@@ -26,6 +17,7 @@ pub(super) fn collect_projected_session_reclamation(
max_deletions: usize,
deletions: &mut CacheDeletionBatch,
) {
remove_obsolete_deferred_heads(session, cache, Some(excluded_path), release_through, deletions);
let mut planned_bytes = 0_u64;
let mut planned_deletions = 0_usize;
let mut transient_readiness_changed = false;
@@ -335,3 +327,27 @@ pub(super) fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) ->
session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id);
Some(MapEntryDeletion { key: map.cache_key, content_length })
}
/// Only retire an inactive deferred FIFO head once its origin binding is gone
/// and every existing working-set protection has released it.
pub(super) fn remove_obsolete_deferred_heads(
session: &mut HlsSession,
cache: &HlsSegmentCache,
excluded: Option<&std::path::Path>,
release_through: Option<u64>,
deletions: &mut CacheDeletionBatch,
) {
while deletions.has_capacity() {
let Some((&seq, entry)) = session.segments.first_key_value() else { break };
let protected = protected_set_for_cache_reclamation(session, cache, excluded, release_through);
if entry.origin_fetch_ref.is_some()
|| !matches!(entry.status, SegmentCacheStatus::CapacityDeferred { .. })
|| protected.segment_proxy_seqs.contains(&seq)
{
break;
}
if let Some(key) = remove_segment_entry(session, seq) {
deletions.push(CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration });
}
}
}
+1
View File
@@ -251,5 +251,6 @@ mod playlist;
mod policy;
mod publication;
mod recovery;
mod revisions;
mod streaming;
mod transport;
+101
View File
@@ -191,3 +191,104 @@ async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the
assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none());
assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some());
}
#[test]
fn garbage_collection_preserves_playback_reserve_and_cutover_deadline() {
use crate::{
media_reserve::{
evaluate_lease_reserve, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackCursor,
HlsLeaseReserveInput, HlsManifestCommitIdentity, HlsManifestDeliveryMode,
},
recovery_timing::{HlsLeaseCutoverTiming, HlsRecoveryTriggerBudgetMs},
terminal_tail::HlsMediaContainer,
};
use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome};
let mut session = HlsSession::new(crate::HlsSessionKey::new(1, "gc-probe"), b"probe", 1_000);
let mut body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n");
for seq in 0..9 {
write!(body, "#EXTINF:12.0,\n{seq}.ts\n").expect("test manifest writes");
}
let OriginManifestParseOutcome::Normal(origin) =
parse_origin_media_manifest(&body, "http://origin.example/index.m3u8")
else {
panic!("normal manifest")
};
session.apply_origin_manifest(&origin).unwrap();
for segment in session.segments.values_mut() {
segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 };
}
session.publishable_origin_head_proxy_seq = Some(4);
let mut cursor = HlsLeasePlaybackCursor::default();
for seq in 0..8 {
let token = cursor.record_request_started(seq, 1_000 + seq * 10_000);
let token = cursor.bind_request_timing(token, &session.segments[&seq]);
let _ = cursor.record_request_completed(token, 1_000 + seq * 10_000);
}
let manifest = HlsLeaseManifestSnapshot {
startup_revisions: None,
delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline,
source_commit_identity: HlsManifestCommitIdentity::new(1),
uri_materialization: None,
finalized_transient_manifest_generation: None,
snapshot_generation: 1,
delivered_at_ms: 71_000,
first_proxy_seq: 4,
last_proxy_seq: 6,
visible_segments: (4..7)
.map(|seq| HlsLeaseManifestSegment {
proxy_seq: seq,
duration_ms: 12_000,
uri: format!("{seq}.ts").into(),
discontinuity_before: false,
map_ref_ready: true,
encryption: None,
})
.collect::<Vec<_>>()
.into(),
discontinuity_sequence: 0,
target_duration_ms: 12_000,
playlist_duration_ms: 36_000,
last_visible_media_end_ms: 36_000,
active_map: None,
active_encryption: None,
container: HlsMediaContainer::MpegTs,
};
let evaluate = |session: &HlsSession| {
let timeline = session.ready_timeline_snapshot(cursor.ready_timeline_start_proxy_seq(4), 71_000);
let reserve = evaluate_lease_reserve(HlsLeaseReserveInput {
manifest: &manifest,
cursor: &cursor,
ready_timeline: &timeline,
now_ms: 71_000,
playback_rate_guard_milli: 1_050,
recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(4_000),
origin_path_degraded: true,
recovery_committed: false,
});
(timeline.units.len(), reserve)
};
let (before_units, before) = evaluate(&session);
assert_eq!(before.guaranteed_reserve_ms, 34_500);
assert!(!ProtectedSet::from_session(&session).segment_proxy_seqs.contains(&0));
// Run the actual metadata-removal function used by garbage collection.
assert!(remove_segment_entry(&mut session, 0).is_some());
let (after_units, after) = evaluate(&session);
let still_ready = session.ready_timeline_snapshot(4, 71_000);
assert_eq!(still_ready.units.len(), 5);
assert_eq!(cursor.first_requested_proxy_seq, Some(0));
assert_eq!(before_units, 5);
assert_eq!(after_units, 5);
assert_eq!(after, before);
let timing =
HlsLeaseCutoverTiming::from_reserve(71_000, after.guaranteed_reserve_ms, after.transition_margin, None);
assert_eq!(timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 93_500);
// Removing the entire historical prefix must keep exactly the same media clock.
for seq in 1..4 {
assert!(remove_segment_entry(&mut session, seq).is_some());
}
assert_eq!(evaluate(&session).1, before);
// A real hole in the current READY span must still shorten the reserve.
assert!(remove_segment_entry(&mut session, 7).is_some());
assert_eq!(evaluate(&session).1.guaranteed_reserve_ms, 10_500);
}
+248
View File
@@ -0,0 +1,248 @@
use super::*;
use crate::{SegmentRevisionKind, SegmentRevisionStore};
#[tokio::test]
async fn diagnostic_inventory_errors_do_not_fail_capacity_reclamation() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let id = session.read().await.proxy_session_id.clone();
let key = crate::SegmentCacheKey::new(id.clone(), 1, "ts");
assert_eq!(gc.cache.capacity_usage(&id).await?.global_bytes, 0);
tokio::fs::write(gc.cache.object_path(&key).parent().unwrap(), b"not a directory").await?;
assert!(gc.cache.session_revision_disk_bytes(&id).await.is_err());
gc.log_capacity_inventory(&id, &GarbageCollectionReport::default()).await;
let outcome = gc
.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest {
proxy_session_id: id.clone(),
target_path: gc.cache.object_path(&key),
required_session_bytes: 0,
required_global_bytes: 0,
staged_bytes: 1,
})
.await?;
assert_eq!(outcome, crate::HlsCacheCapacityReclaimOutcome::default());
let occupied = crate::SegmentCacheKey::new(ProxySessionId("occupied".into()), 1, "ts");
gc.cache.write_bytes_and_commit(&occupied, &[1; 20]).await?;
gc.cache.update_cache_limits(30, 30);
let error = gc.cache.reserve_revision_peak_reclaiming(&key, 20).await.err().unwrap();
assert!(
crate::cache::hls_cache_capacity_from_io(&error).is_some(),
"diagnostic I/O must preserve the functional capacity error"
);
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
Ok(())
}
#[tokio::test]
async fn global_reclamation_retires_revisions_unpinned_by_another_sessions_segment_deletion() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, old_session) = gc_with_session(&directory).await;
let incoming = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "incoming"), b"secret", 1).await;
let old_id = old_session.read().await.proxy_session_id.clone();
let incoming_id = incoming.read().await.proxy_session_id.clone();
let store = Arc::new(SegmentRevisionStore::default());
gc.install_revision_store(&store);
let revision = store.create(old_id.clone(), 1, SegmentRevisionKind::Processed)?;
let revision_key = revision.revision().key.clone();
let metadata = gc.cache.write_bytes_and_commit(&revision_key, &[7; 20]).await?;
revision.revision().complete(metadata);
*revision.revision().file_pin.lock().unwrap() =
Some(gc.cache.pin_revision_file(&gc.cache.object_path(&revision_key))?);
let old_key = {
let mut session = old_session.write().await;
apply_six_segment_manifest_for_gc(&mut session);
session.publishable_origin_head_proxy_seq = Some(2);
let head = session.segments.get_mut(&1).unwrap();
head.origin_fetch_ref = None;
head.status = SegmentCacheStatus::Ready { content_length: 20, ready_at_ms: 0 };
let key = head.cache_key.clone();
let config = tuliprox_core::model::HlsStartupConfig::default();
session.startup = Some(crate::HlsSessionStartup {
config: config.clone(),
first_data_timeout_ms: 1_000,
worker: std::sync::Weak::new(),
access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())),
revisions: [(1, revision)].into(),
policy_fixed: true,
store,
budget: crate::ProgressiveBudgetManager::new(config),
});
key
};
gc.cache.write_bytes_and_commit(&old_key, &[1; 20]).await?;
gc.cache.update_cache_limits(50, 50);
assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40);
let key = crate::SegmentCacheKey::new(incoming_id.clone(), 1, "ts");
let peak = Arc::new(
tokio::time::timeout(Duration::from_secs(2), gc.cache.reserve_revision_peak_reclaiming(&key, 40))
.await
.expect("global reclamation must finish without a periodic GC")?,
);
assert!(gc.cache.metadata(&old_key).await?.is_none());
assert!(gc.cache.metadata(&revision_key).await?.is_none());
assert_eq!(gc.cache.capacity_usage(&old_id).await?.session_bytes, 0);
assert!(old_session.read().await.startup.as_ref().unwrap().revisions.is_empty());
let staged = gc
.cache
.stage_temp_with_budget(
&key,
&[9; 40][..],
tokio::time::Instant::now() + Duration::from_secs(2),
Some(Arc::clone(&peak)),
)
.await?;
gc.cache.commit_staged(&key, staged).await?;
drop(peak);
assert_eq!(gc.cache.remaining_revision_bytes(&incoming_id), 0);
assert_eq!(gc.cache.capacity_usage(&incoming_id).await?.global_bytes, 40);
assert_eq!(tokio::fs::read(gc.cache.object_path(&key)).await?, [9; 40]);
Ok(())
}
#[tokio::test]
async fn targeted_reclamation_deletes_only_unpinned_revisions_of_the_same_session() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let id = session.read().await.proxy_session_id.clone();
let store = Arc::new(SegmentRevisionStore::default());
gc.install_revision_store(&store);
let metrics = Arc::new(crate::HlsCacheMetrics::default());
gc.cache.install_revision_metrics(&metrics);
let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?;
gc.cache.write_bytes_and_commit(&revision.revision().key, &[7; 80]).await?;
let reader = revision.clone();
drop(revision);
gc.cache.update_cache_limits(100, 100);
let next = crate::SegmentCacheKey::new(id.clone(), 1, "ts");
assert!(gc.cache.reserve_revision_peak_reclaiming(&next, 30).await.is_err());
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
drop(reader);
let peak = gc.cache.reserve_revision_peak_reclaiming(&next, 30).await?;
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0);
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
assert_eq!(metrics.snapshot().revision_disk_bytes, 0);
drop(peak);
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
assert_eq!(gc.run_once(0).await?.revision_deletions_succeeded, 0);
Ok(())
}
#[tokio::test]
async fn failed_revision_deletes_preserve_bytes_and_are_retried_idempotently() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let id = session.read().await.proxy_session_id.clone();
let store = Arc::new(SegmentRevisionStore::default());
gc.install_revision_store(&store);
let revision = store.create(id.clone(), 0, SegmentRevisionKind::Processed)?;
let key = revision.revision().key.clone();
gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?;
gc.cache.fail_next_cache_deletes(gc.cache.object_path(&key), 2);
drop(revision);
let first = gc.run_once(0).await?;
assert_eq!(first.revision_deletions_failed, 2);
assert_eq!(first.revision_bytes_reclaimed, 0);
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
let next = gc.run_once(0).await?;
assert_eq!(next.revision_deletions_succeeded, 1);
assert_eq!(next.revision_bytes_reclaimed, 80);
assert!(!tokio::fs::try_exists(gc.cache.object_path(&key)).await?);
assert_eq!(gc.run_once(0).await?.revision_deletions_planned, 0);
Ok(())
}
#[tokio::test]
async fn obsolete_deferred_fifo_head_retires_without_skipping_a_protected_head() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let key = {
let mut session = session.write().await;
apply_six_segment_manifest_for_gc(&mut session);
let head = session.segments.get_mut(&1).unwrap();
head.status =
SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Demand, deferred_at_ms: 1 };
head.origin_fetch_ref = None;
head.access.reader_started(1);
head.cache_key.clone()
};
gc.cache.write_bytes_and_commit(&key, b"partial-commit").await?;
let first = gc.run_once(0).await?;
assert_eq!(first.segments_deleted(), 0);
assert!(session.read().await.segments.contains_key(&1));
session.write().await.segments.get_mut(&1).unwrap().access.reader_finished();
let second = gc.run_once(0).await?;
assert_eq!(second.segments_deleted(), 1);
assert!(!session.read().await.segments.contains_key(&1));
assert!(gc.cache.metadata(&key).await?.is_none());
assert_eq!(gc.run_once(0).await?.segments_deleted(), 0);
Ok(())
}
#[tokio::test]
async fn revision_cleanup_skips_contended_locks_and_pressure_keeps_protection_unknown() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let id = session.read().await.proxy_session_id.clone();
let leases = Arc::new(tokio::sync::RwLock::new(crate::HlsAccessLeaseStore::default()));
gc.install_access_leases(&leases);
let lease_guard = leases.write().await;
let mut report = GarbageCollectionReport::default();
tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report))
.await
.expect("revision cleanup must not wait while the lease lock is held")?;
assert_eq!(report.revision_lock_conflicts, 1);
drop(lease_guard);
let key = crate::SegmentCacheKey::new(id.clone(), 0, "ts");
gc.cache.write_bytes_and_commit(&key, &[7; 80]).await?;
let session_guard = session.write().await;
let mut report = GarbageCollectionReport::default();
tokio::time::timeout(Duration::from_secs(2), gc.cleanup_revisions(&[Arc::clone(&session)], &mut report))
.await
.expect("revision cleanup must not reverse the lease/session lock order")?;
assert_eq!(report.revision_lock_conflicts, 1);
let outcome = tokio::time::timeout(
Duration::from_secs(2),
gc.reclaim_for_projected_write(crate::HlsCacheCapacityReclaimRequest {
proxy_session_id: id.clone(),
target_path: gc.cache.object_path(&key),
required_session_bytes: 1,
required_global_bytes: 0,
staged_bytes: 1,
}),
)
.await
.expect("targeted selection must not wait for a contended session")?;
assert_eq!(outcome.protected_working_set_bytes, 0, "a lock conflict cannot claim all 80 bytes are protected");
assert_eq!(outcome.reclaimed_session_bytes, 0);
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 80);
drop(session_guard);
Ok(())
}
#[tokio::test]
async fn shared_revision_cleanup_retires_a_bounded_batch_and_drains_idempotently() -> std::io::Result<()> {
let directory = tempfile::tempdir()?;
let (gc, session) = gc_with_session(&directory).await;
let id = session.read().await.proxy_session_id.clone();
let store = Arc::new(SegmentRevisionStore::default());
gc.install_revision_store(&store);
let limit = super::super::MAX_CACHE_DELETE_RETRIES_PER_RUN;
for seq in 0..limit + 3 {
let revision = store.create(id.clone(), u64::try_from(seq).unwrap(), SegmentRevisionKind::Processed)?;
gc.cache.write_bytes_and_commit(&revision.revision().key, &[7]).await?;
}
for expected in [limit, 3, 0] {
let mut report = GarbageCollectionReport::default();
gc.cleanup_revisions(&[Arc::clone(&session)], &mut report).await?;
assert_eq!(report.revision_deletions_planned, expected);
assert_eq!(report.revision_deletions_succeeded, expected);
assert_eq!(report.revision_deletions_failed, 0);
assert_eq!(report.revision_bytes_reclaimed, u64::try_from(expected).unwrap());
assert_eq!(gc.cache.revision_disk_bytes(), gc.cache.scan_revision_disk_bytes().await?);
}
assert_eq!(gc.cache.capacity_usage(&id).await?.session_bytes, 0);
assert_eq!(gc.cache.remaining_revision_bytes(&id), 0);
Ok(())
}
+4
View File
@@ -68,8 +68,12 @@ async fn global_size_gc_subtracts_map_bytes_after_segment_removal() {
session.apply_origin_manifest(&manifest).expect("manifest should map");
for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] {
let segment = session.segments.get_mut(&proxy_seq).expect("segment");
gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("segment fixture");
segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms };
}
for map in session.maps.values() {
gc.cache.write_bytes_and_commit(&map.cache_key, &[0; 10]).await.expect("map fixture");
}
session.maps.get_mut(&ProxyMapId(0)).expect("first map").status =
MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 };
session.maps.get_mut(&ProxyMapId(1)).expect("second map").status =
+18
View File
@@ -268,3 +268,21 @@ mod tests {
}
}
}
#[cfg(test)]
mod short_window_tests {
use super::*;
#[test]
fn strip_preserves_every_segment_of_an_already_short_playlist() {
for count in [1, 2] {
for mode in [HlsStripMode::Segments, HlsStripMode::Seconds] {
let strip = StripConfig { mode, value: 30 };
let body = format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n{}", "#EXTINF:2.5,\nsegment.ts\n".repeat(count));
let view = materialize_initial_hls_strip_view(&body, &strip, HlsManifestWindowPolicy::ApplyLiveWindow);
assert_eq!(view.body, body);
assert_eq!(initial_hls_strip_segments_for_durations(&strip, &vec![2_500; count]), 0);
}
}
}
}
+168
View File
@@ -0,0 +1,168 @@
//! Finite custom media published after the live splice deadline has elapsed.
use crate::{
hls_ctx::HlsCtx,
lease::{HlsAccessLeaseId, HlsTerminalTailPreparation},
prepared_terminal_bundle::{prepared_terminal_bundle_key, HlsPreparedTerminalBundle},
runtime_custom_tail::{
await_prepared_standalone_bundle, snapshot_hls_runtime_custom_tail_asset, standalone_custom_asset_fingerprint,
HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsStandaloneCustomPlan,
HlsStandaloneCustomPlanKey,
},
terminal_tail::{HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailGeneration},
HlsAccessLeaseState, ProxySessionId,
};
use std::{fmt, fmt::Write, sync::Arc};
use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs};
/// Immutable, lease-bound custom playlist that does not consume a live splice base.
#[derive(Clone)]
pub struct HlsLateCustomResponsePlan {
pub generation: HlsTerminalTailGeneration,
pub(crate) manifest_snapshot_generation: u64,
pub(crate) key: HlsStandaloneCustomPlanKey,
proxy_session_id: ProxySessionId,
lease_id: HlsAccessLeaseId,
standalone: HlsStandaloneCustomPlan,
}
impl HlsLateCustomResponsePlan {
pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool {
self.proxy_session_id == *proxy_session_id && self.lease_id == *lease_id
}
pub fn manifest_body(&self) -> &str { &self.standalone.manifest_body }
pub(crate) fn segment_bytes(
&self,
fingerprint: &str,
index: u16,
) -> Result<bytes::Bytes, crate::runtime_custom_tail::HlsStandaloneCustomSegmentError> {
use crate::runtime_custom_tail::HlsStandaloneCustomSegmentError;
if fingerprint != standalone_custom_asset_fingerprint(self.key.asset) {
return Err(HlsStandaloneCustomSegmentError::StaleAssetFingerprint);
}
self.standalone.segment_bytes(index).ok_or(HlsStandaloneCustomSegmentError::InvalidIndex)
}
}
impl fmt::Debug for HlsLateCustomResponsePlan {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("HlsLateCustomResponsePlan")
.field("generation", &self.generation)
.field("key", &self.key)
.finish_non_exhaustive()
}
}
impl PartialEq for HlsLateCustomResponsePlan {
fn eq(&self, other: &Self) -> bool {
self.generation == other.generation
&& self.manifest_snapshot_generation == other.manifest_snapshot_generation
&& self.key == other.key
&& self.proxy_session_id == other.proxy_session_id
&& self.lease_id == other.lease_id
&& self.standalone.manifest_body == other.standalone.manifest_body
}
}
impl Eq for HlsLateCustomResponsePlan {}
pub(crate) enum HlsLateCustomResponseError {
Superseded,
Unavailable(HlsTerminalTailCompatibility),
}
pub(crate) async fn prepare_late_custom_response(
ctx: &HlsCtx,
proxy_session_id: &ProxySessionId,
lease_id: &HlsAccessLeaseId,
preparation: &HlsTerminalTailPreparation,
now_ms: u64,
) -> Result<(Arc<HlsLateCustomResponsePlan>, HlsRuntimeCustomTailAssetIdentity), HlsLateCustomResponseError> {
let lease = ctx
.hls_proxy
.access_lease_response_snapshot(lease_id, proxy_session_id, now_ms)
.await
.ok_or(HlsLateCustomResponseError::Superseded)?;
if !matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle)
|| lease.playback_mode != HlsLeasePlaybackMode::Live
|| lease.issued_at_ms != preparation.lease_issued_at_ms
|| lease.admission_generation != preparation.expected_admission_generation
|| lease.playback_cursor.cursor_generation != preparation.cursor_generation
|| lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation)
!= Some(preparation.manifest_snapshot_generation)
{
return Err(HlsLateCustomResponseError::Superseded);
}
let snapshot = snapshot_hls_runtime_custom_tail_asset(ctx, HlsRuntimeCustomTailReason::ChannelUnavailable)
.map_err(|error| HlsLateCustomResponseError::Unavailable(error.terminal_compatibility()))?;
let expected_asset = HlsRuntimeCustomTailAssetIdentity::from_asset(&snapshot);
let bundle_key = prepared_terminal_bundle_key(
&snapshot.asset,
preparation.manifest_snapshot.target_duration_ms,
crate::terminal_tail::HLS_TERMINAL_TAIL_SEGMENT_COUNT,
);
let bundle = await_prepared_standalone_bundle(ctx, Arc::clone(&snapshot.asset), bundle_key)
.await
.map_err(HlsLateCustomResponseError::Unavailable)?;
let first_proxy_seq = lease
.published_media_sequence_highwater
.unwrap_or(preparation.manifest_snapshot.last_proxy_seq)
.checked_add(1)
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?;
let discontinuity_sequence =
lease.published_discontinuity_sequence_highwater.max(preparation.manifest_snapshot.discontinuity_sequence);
let path_prefix = preparation
.manifest_snapshot
.visible_segments
.first()
.and_then(|segment| {
let uri = preparation.manifest_snapshot.materialize_uri(&segment.uri);
uri.find("/hls/shared/live/").map(|offset| uri[..offset].to_owned())
})
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::InvalidLeaseRoute))?;
let manifest_body = render_late_manifest(first_proxy_seq, discontinuity_sequence, &path_prefix, lease_id, &bundle)?;
let key = HlsStandaloneCustomPlanKey {
reason: snapshot.reason,
asset: expected_asset.media,
segment_count: bundle.key.segment_count,
};
let standalone = HlsStandaloneCustomPlan { key, manifest_body: Arc::from(manifest_body), prepared_bundle: bundle };
Ok((
Arc::new(HlsLateCustomResponsePlan {
generation: HlsTerminalTailGeneration(preparation.decision_generation),
manifest_snapshot_generation: preparation.manifest_snapshot_generation,
key,
proxy_session_id: proxy_session_id.clone(),
lease_id: lease_id.clone(),
standalone,
}),
expected_asset,
))
}
// Keeping the layout separate makes the publication boundary testable without a live origin or cache.
fn render_late_manifest(
first_proxy_seq: u64,
discontinuity_sequence: u64,
path_prefix: &str,
lease_id: &HlsAccessLeaseId,
bundle: &HlsPreparedTerminalBundle,
) -> Result<String, HlsLateCustomResponseError> {
first_proxy_seq
.checked_add(u64::from(bundle.key.segment_count).saturating_sub(1))
.ok_or(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))?;
let fingerprint = standalone_custom_asset_fingerprint(bundle.key.asset);
let extinf = format_hls_duration_ms(bundle.source_asset_duration_ms);
let mut manifest = format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:{}\n#EXT-X-MEDIA-SEQUENCE:{first_proxy_seq}\n#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuity_sequence}\n#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n", hls_target_duration_secs(bundle.key.target_duration_ms));
for index in 0..bundle.key.segment_count {
let _ = writeln!(manifest, "#EXTINF:{extinf},\n{path_prefix}/cvs/hls/{}/{fingerprint}/{index}.ts", lease_id.0);
}
manifest.push_str("#EXT-X-ENDLIST\n");
Ok(manifest)
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,47 @@
use super::*;
use crate::{
prepared_terminal_bundle::{HlsPreparedTerminalBundleKey, HlsPreparedTerminalSegment},
terminal_tail::HlsTerminalAssetIdentity,
};
fn bundle() -> HlsPreparedTerminalBundle {
HlsPreparedTerminalBundle {
key: HlsPreparedTerminalBundleKey {
asset: HlsTerminalAssetIdentity { revision: 7, fingerprint: [7; 32] },
target_duration_ms: 12_000,
segment_count: 12,
},
source_asset_duration_ms: 10_027,
source_asset_duration_ticks_90khz: 902_400,
segments: (0..12)
.map(|index| HlsPreparedTerminalSegment {
index,
timestamp_offset_ticks_90khz: u64::from(index) * 902_400,
bytes: bytes::Bytes::from_static(b"prepared terminal segment"),
})
.collect(),
}
}
#[test]
fn late_manifest_keeps_target_and_sequences_and_contains_only_custom_media() {
let bundle = bundle();
let manifest = render_late_manifest(131, 9, "/iptv", &HlsAccessLeaseId("lease".into()), &bundle)
.unwrap_or_else(|_| panic!("valid late manifest"));
assert!(manifest.contains("#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:131\n#EXT-X-DISCONTINUITY-SEQUENCE:9\n"));
assert_eq!(manifest.matches("#EXTINF:10.027,").count(), 12);
assert_eq!(manifest.matches("#EXT-X-DISCONTINUITY\n").count(), 1);
assert!(manifest.contains("#EXT-X-KEY:METHOD=NONE\n"));
assert!(manifest.contains("/iptv/cvs/hls/lease/0000000000000007/11.ts"));
assert!(!manifest.contains("/hls/shared/live/"));
assert!(manifest.ends_with("#EXT-X-ENDLIST\n"));
}
#[test]
fn late_manifest_rejects_sequence_overflow_instead_of_reusing_live_sequences() {
let bundle = bundle();
assert!(matches!(
render_late_manifest(u64::MAX - 1, 0, "", &HlsAccessLeaseId("lease".into()), &bundle,),
Err(HlsLateCustomResponseError::Unavailable(HlsTerminalTailCompatibility::ManifestRenderFailed))
));
}
+2
View File
@@ -144,6 +144,8 @@ pub struct HlsAccessLease {
pub startup_admission: HlsLeaseStartupAdmissionState,
pub playback_cursor: HlsLeasePlaybackCursor,
pub last_manifest_snapshot: Option<HlsLeaseManifestSnapshot>,
pub(crate) published_media_sequence_highwater: Option<u64>,
pub(crate) published_discontinuity_sequence_highwater: u64,
revision_bindings: Option<Arc<HlsLeaseRevisionBindings>>,
progressive_startup_claimed: bool,
published_transient_resource_ids: HlsPublishedTransientResourceIds,
+3 -1
View File
@@ -37,7 +37,9 @@ impl HlsAccessLease {
HlsLeasePlaybackMode::TerminalTail(plan) => {
HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation }
}
HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None,
HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::TerminalUnavailable { .. }
| HlsLeasePlaybackMode::Ended => return None,
};
Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback })
}
+21 -1
View File
@@ -93,7 +93,9 @@ impl HlsAccessLease {
HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => {
Some(HlsTerminalTailGeneration(decision_generation))
}
HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None,
HlsLeasePlaybackMode::Live
| HlsLeasePlaybackMode::TerminalCustomResponse(_)
| HlsLeasePlaybackMode::Ended => None,
};
if self.pending_terminal_protection_release.is_none() {
self.pending_terminal_protection_release = release_generation;
@@ -370,6 +372,24 @@ impl HlsAccessLeaseStore {
Some(token)
}
pub(crate) fn record_timed_segment_request_started_if_identity_matches(
&mut self,
lease_id: &HlsAccessLeaseId,
proxy_session_id: &ProxySessionId,
expected: HlsMediaLeaseIdentity,
segment: &crate::SegmentEntry,
requested_at_ms: u64,
) -> Option<HlsPlaybackRequestToken> {
let token = self.record_segment_request_started_if_identity_matches(
lease_id,
proxy_session_id,
expected,
segment.proxy_seq,
requested_at_ms,
)?;
Some(self.by_lease_id.get_mut(lease_id)?.playback_cursor.bind_request_timing(token, segment))
}
pub fn record_segment_request_completed_if_identity_matches(
&mut self,
lease_id: &HlsAccessLeaseId,

Some files were not shown because too many files have changed in this diff Show More